From 325dae46aeeb2affa5eb5f3b27735bb3fa23759f Mon Sep 17 00:00:00 2001 From: turbocrime Date: Sat, 9 Sep 2023 00:00:13 -0700 Subject: [PATCH] Worker streaming (#3) squash streaming work --- README.md | 41 +-- demo/index.html | 20 +- demo/package.json | 2 +- demo/src/main.ts | 315 ++++++++++++++-------- demo/src/style.css | 9 +- demo/tsconfig.json | 2 +- package.json | 21 +- pnpm-lock.yaml | 156 +++++------ rome.json | 5 +- src/CamCommand.ts | 176 ++++++++++++ src/Camera.ts | 236 ++++++++++++++++ src/{kinectMotor.ts => Motor.ts} | 62 ++--- src/enum/cam.ts | 126 +++++++++ src/enum/index.ts | 2 + src/enum/motor.ts | 21 ++ src/index.ts | 51 +++- src/kinectCamera.ts | 272 ------------------- src/kinectDevice.ts | 83 ------ src/kinectStream.ts | 170 ------------ src/stream/CamFrameAssembler.ts | 55 ++++ src/stream/CamIsoParser.ts | 115 ++++++++ src/stream/CamStream.ts | 112 ++++++++ src/stream/UnderlyingIsochronousSource.ts | 107 ++++++++ src/stream/index.ts | 4 + src/util/frame.ts | 305 +++++++++++++++++++++ src/util/index.ts | 2 + src/util/mode.ts | 146 ++++++++++ src/util/tsconfig.json | 8 + src/worker/CamIsoWorker.ts | 134 +++++++++ src/worker/tsconfig.json | 8 + tsconfig.json | 6 +- 31 files changed, 1968 insertions(+), 804 deletions(-) create mode 100644 src/CamCommand.ts create mode 100644 src/Camera.ts rename src/{kinectMotor.ts => Motor.ts} (52%) create mode 100644 src/enum/cam.ts create mode 100644 src/enum/index.ts create mode 100644 src/enum/motor.ts delete mode 100644 src/kinectCamera.ts delete mode 100644 src/kinectDevice.ts delete mode 100644 src/kinectStream.ts create mode 100644 src/stream/CamFrameAssembler.ts create mode 100644 src/stream/CamIsoParser.ts create mode 100644 src/stream/CamStream.ts create mode 100644 src/stream/UnderlyingIsochronousSource.ts create mode 100644 src/stream/index.ts create mode 100644 src/util/frame.ts create mode 100644 src/util/index.ts create mode 100644 src/util/mode.ts create mode 100644 src/util/tsconfig.json create mode 100644 src/worker/CamIsoWorker.ts create mode 100644 src/worker/tsconfig.json diff --git a/README.md b/README.md index 3326a4e..b658895 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,29 @@ -# webnect *...now with [live demo](https://turbocrime.github.io/webnect/)!* +# webnect + +**(try [live demo](https://turbocrime.github.io/webnect/))** this is a webusb driver for microsoft's xbox360 kinect. -![webnect](https://github.com/turbocrime/webnect/assets/134443988/1bfbb58f-4a5a-4276-8cde-b80c7d91b63a) + chrome only :'C mozzy dont do webusb i have never written a usb driver before, nor even a typescript library, so critique is welcome. -currently, this driver only supports "Xbox NUI Motor" PID `0x02b0` and "Xbox NUI Camera" PID `0x02ae` devices, labelled "Model 1414", because that's what i found at goodwill. i believe there's a few externally-identical models of "kinect", some with dramatic hardware revisions. if your device doesn't work with this, please verify that it works at all, and then send me the details. +this driver at least works with "Xbox NUI Motor" PID `0x02b0` and "Xbox NUI Camera" PID `0x02ae` devices, labelled "Model 1414", because that's what i found at goodwill. i there may be a few externally-identical models of "kinect", some with dramatic hardware revisions. if your device doesn't work with this, please verify that it works at all, and then send me the details. ## what the kinect is an early consumer depth sensor based on structured light projection, plus some other goodies. it was released in 2010 as a gamer thing and nobody cares about it anymore except me -they're fun, and they go for like $5 now. plus they're usb2, so i can drive them with throwaway SBCs like an rpi3. maybe not with this driver, but that's how i got familiar. i been using them for video synth input, and interactive generative art installations (yes i do parties, hmu) but it's the kind of thing you really gotta see in person +they're fun, and they go for like $5 now. plus they're usb2, so i can drive them with throwaway SBCs like an rpi3. maybe not with this driver, but that's how i got familiar. i been using them for video synth input, and interactive generative art installations a webusb driver lets more folks see it in person :) after i rewrite everything :) ## ware +original kinect only. + ### Xbox NUI Motor * accelerometer @@ -28,10 +32,11 @@ a webusb driver lets more folks see it in person :) after i rewrite everything : ### Xbox NUI Camera -* depth camera 11bit only -* thats it for now -* no ir -* no visible +* depth 11bpp, 10bpp +* visible 8bpp bayer +* infrared 10bpp + +visible and infrared stream from the same endpoint, so you can only have one at a time. ## why @@ -41,11 +46,13 @@ whatever. its the future and webusb is real ## how -available on npm as [`@webnect/webnect`](https://www.npmjs.com/package/@webnect/webnect) +go dig your kinect out of the closet. plug it in. open + +## diy -for local demos, clone the repo. +available on npm as [`@webnect/webnect`](https://www.npmjs.com/package/@webnect/webnect) -go dig your kinect out of the closet. plug it in. run +or for a local demo, clone this repo. run ```sh $ pnpm install @@ -79,24 +86,24 @@ devices? : { pass a boolean indicating your desire to request acquisition, or pass a USBDevice if you have already one already acquired. -## it dont work +### um its broekn if you see an empty device selection modal, you probably have the wrong model kinect. you can check your usb devices with `lsusb` on linux or on `system_profiler SPUSBDataType` on macos if you see glitchy stream output, haha nice. cool -## bad parts +### bad parts a single kinect is technically three devices in a trenchcoat. afaict there's no way to associate them, because webusb won't expose bus position. it doesn't matter; you probably only plugged in one kinect anyway. also, typescript aint exactly the optimal language for bitmath or destructuring binary data -## the future +## way -the mathy parts are an obvious candidate for webgpu acceleration. do NOT send a patch i wanna do it +the mathy parts are an obvious candidate for assemblyscript and webgpu acceleration. do NOT send a patch i wanna do it -i should probably learn how to actually use canvas and arraybuffers +i should probably learn how to actually use canvas and streams -probably going after ir video next, then bayer/yuv, and then audio device and firmware stuff. +probably going after pose features next, maybe registration of visible light to depth frames. and then audio device and firmware stuff. someday.... kinect2? diff --git a/demo/index.html b/demo/index.html index 8651ba8..6ff7132 100644 --- a/demo/index.html +++ b/demo/index.html @@ -4,7 +4,7 @@ - kinect WebUSB demo + webnect demo @@ -19,6 +19,10 @@

webnect is a WebUSB driver f

when your kinect is plugged in, activate it with the UI below

an image of the appropriate kinect +

Firefox, Safari are not supported ;_; only chrom

i have only tried this with my kinect. you can try it on yours if you like

@@ -50,12 +54,16 @@

Camera

 - - + + +
FPS:
+
- +

diff --git a/demo/package.json b/demo/package.json index 0e82760..a725f8d 100644 --- a/demo/package.json +++ b/demo/package.json @@ -9,7 +9,7 @@ "preview": "vite preview" }, "dependencies": { - "@webnect/webnect": "workspace:../" + "@webnect/webnect/": "workspace:../" }, "devDependencies": { "@types/node": "^20.4.4", diff --git a/demo/src/main.ts b/demo/src/main.ts index a0af685..9e8b037 100644 --- a/demo/src/main.ts +++ b/demo/src/main.ts @@ -1,66 +1,82 @@ import "./style.css"; import { - KinectDevice, - KinectCamera, - KinectProductId, - usbSupport, + claimNuiCamera, + claimNuiMotor, + ProductId, + Camera, + Motor, } from "@webnect/webnect"; -if (usbSupport) document.getElementById("annoying")!.remove(); +import { unpackGray, DefaultModes, modes } from "@webnect/webnect/util"; + +const customDepthRgba = (raw: ArrayBuffer, rgba?: Uint8ClampedArray) => { + const rgbaFrame = rgba ?? new Uint8ClampedArray(640 * 480 * 4); + // frame is 11bit/u16gray, expand for canvas rgba + const grayFrame = unpackGray(11, raw); + + // moving color ramps + const colorMarch = window.performance.now() / 10; + for (let i = 0; i < grayFrame.length && i * 4 < rgbaFrame.length; i++) { + const grayPixel = grayFrame[i]; + + // this counts as art + rgbaFrame[i * 4 + 0] = ((grayPixel << 1) + colorMarch) & 0xff; + rgbaFrame[i * 4 + 1] = ((grayPixel << 2) + colorMarch) & 0xff; + rgbaFrame[i * 4 + 2] = ((grayPixel << 3) + colorMarch) & 0xff; + rgbaFrame[i * 4 + 3] = grayPixel < 2047 ? 0xff : 0x00; + } + return rgbaFrame; +}; + +if (typeof navigator?.usb?.getDevices === "function") + document.getElementById("annoying")!.remove(); let existingUsb = await navigator.usb.getDevices(); -const forgottenUsb = ["Reload page to reconnect to forgotten devices"]; +const forgottenUsb = new Array(); +const plugItIn = document.querySelector("#plugItIn")!; +const pairedDeviceDisplay = + document.querySelector("#pairedDevices")!; function renderExistingUsb() { - const plugItIn = document.querySelector("#plugItIn")!; - const pairedDeviceDisplay = - document.querySelector("#pairedDevices")!; - if (!existingUsb.length && forgottenUsb.length < 2) { - plugItIn.hidden = false; - pairedDeviceDisplay.hidden = true; - } else { - plugItIn.hidden = true; - pairedDeviceDisplay.hidden = false; - const pairedDeviceList = - document.querySelector("#pairedDeviceList")!; - pairedDeviceList.innerHTML = ""; - existingUsb.forEach((device) => { - const idStr = [device.productName!, device?.serialNumber].join(" "); + const pairedDeviceList = + document.querySelector("#pairedDeviceList")!; + pairedDeviceList.innerHTML = ""; + existingUsb.forEach((device) => { + const idStr = [device.productName!, device?.serialNumber].join(" "); - const option = Object.assign(document.createElement("label"), { - className: "pairedDevice", - textContent: idStr, - }); - const checkbox = Object.assign(document.createElement("input"), { - type: "checkbox", - checked: true, - value: idStr, - }); + const option = Object.assign(document.createElement("label"), { + className: "pairedDevice", + textContent: idStr, + }); + const checkbox = Object.assign(document.createElement("input"), { + type: "checkbox", + checked: true, + value: idStr, + }); - option.prepend(checkbox); - checkbox.addEventListener("change", async () => { - await device.close(); - await device.forget(); - forgottenUsb.push(idStr); - updateExistingUsb(); - }); - pairedDeviceList.appendChild(option); + option.prepend(checkbox); + checkbox.addEventListener("change", async () => { + await device.close(); + await device.forget(); + forgottenUsb.push(idStr); + updateExistingUsb(); }); - forgottenUsb.forEach((forgottenStr) => { - const forgottenOption = Object.assign(document.createElement("label"), { - className: "pairedDevice forgotten", - textContent: forgottenStr, - }); - const disabledCheckBox = Object.assign(document.createElement("input"), { - type: "checkbox", - checked: false, - disabled: true, - }); - forgottenOption.prepend(disabledCheckBox); - pairedDeviceList.appendChild(forgottenOption); + pairedDeviceList.appendChild(option); + }); + forgottenUsb.forEach((forgottenStr) => { + const forgottenOption = Object.assign(document.createElement("label"), { + className: "pairedDevice forgotten", + textContent: forgottenStr, }); - } + const disabledCheckBox = Object.assign(document.createElement("input"), { + type: "checkbox", + checked: false, + disabled: true, + }); + forgottenOption.prepend(disabledCheckBox); + pairedDeviceList.appendChild(forgottenOption); + }); } const updateExistingUsb = async () => { @@ -68,14 +84,11 @@ const updateExistingUsb = async () => { renderExistingUsb(); }; -function setupKinect(requestUsbBtn: HTMLButtonElement) { - const activateDemos = (k: KinectDevice) => { - if (k.motor) setupMotorDemo(k); - if (k.camera) setupDepthDemo(k); - }; - +async function setupDevice() { + await updateExistingUsb(); if (existingUsb.length) { - renderExistingUsb(); + plugItIn.hidden = true; + pairedDeviceDisplay.hidden = false; const devicesArg: { motor: boolean | USBDevice; camera: boolean | USBDevice; @@ -83,125 +96,189 @@ function setupKinect(requestUsbBtn: HTMLButtonElement) { } = { motor: false, camera: false, audio: false }; existingUsb.forEach((device) => { switch (device.productId) { - case KinectProductId.NUI_MOTOR: + case ProductId.NUI_MOTOR: devicesArg.motor = device; break; - case KinectProductId.NUI_CAMERA: + case ProductId.NUI_CAMERA: devicesArg.camera = device; break; - case KinectProductId.NUI_AUDIO: - devicesArg.audio = device; - break; } }); - new KinectDevice(devicesArg).ready.then(activateDemos); + if (devicesArg.camera) setupCameraDemo(devicesArg.camera as USBDevice); + if (devicesArg.motor) setupMotorDemo(devicesArg.motor as USBDevice); } +} +setupDevice(); - requestUsbBtn.addEventListener("click", () => { +document + .querySelector("#connectUsb")! + .addEventListener("click", async () => { const { motor, camera, audio } = { - motor: document.querySelector("#motorCb")!.checked, - camera: document.querySelector("#cameraCb")!.checked, + motor: document.querySelector("#motorCb")!.checked + ? await claimNuiMotor() + : undefined, + camera: document.querySelector("#cameraCb")!.checked + ? await claimNuiCamera() + : undefined, audio: document.querySelector("#audioCb")!.checked, }; if (!(motor || camera || audio)) return alert("Select at least one device."); - new KinectDevice({ motor, camera, audio }).ready - .then(activateDemos) - .then(updateExistingUsb); + setupDevice(); }); -} - -setupKinect(document.querySelector("#connectUsb")!); -function setupDepthDemo(kinect: KinectDevice) { +function setupCameraDemo(cameraDevice: USBDevice) { + cameraDevice.open(); + const camera = new Camera(cameraDevice); const cameraDemo = document.querySelector("#cameraDemo")!; cameraDemo.hidden = false; cameraDemo.disabled = false; cameraDemo.classList.remove("disabled"); - const depthStreamCb = - document.querySelector("#depthStream")!; - const depthCanvas = - document.querySelector("#depthCanvas")!; - const depthCtx = depthCanvas.getContext("2d")!; + const frameCounter = Array(); + const videoFps = document.getElementById("videoFps")!; + setInterval(() => { + videoFps.innerText = `${frameCounter.length}`; + frameCounter.splice(0, frameCounter.length); + }, 1000); + + const videoCanvas = + document.querySelector("#videoCanvas")!; + const videoCanvas2dCtx = videoCanvas.getContext("2d")!; + // calculate fullscreen center crop const canvasAspect = 640 / 480; const screenAspect = screen.width / screen.height; - // center crop fullscreen const fsWidth = screenAspect > canvasAspect ? 640 : screenAspect * 480; const fsHeight = screenAspect > canvasAspect ? 640 / screenAspect : 480; const fsZeroX = -((640 - fsWidth) / 2); const fsZeroY = -((480 - fsHeight) / 2); - const fsDepth = document.querySelector("#fsDepth")!; - fsDepth.addEventListener("click", () => { - depthCanvas.requestFullscreen(); + const videoModeOption = + document.querySelector("#videoMode")!; + videoModeOption.addEventListener("change", async () => { + await endStream(); + console.log("ended stream"); + await runStream(); + }); + + const videoFlipCb = document.querySelector("#flipCb")!; + videoFlipCb.addEventListener("change", async () => { + camera.setMode( + modes( + { flip: videoFlipCb.checked ? 1 : 0 }, + { flip: videoFlipCb.checked ? 1 : 0 }, + ), + ); + }); + + const videoFsBtn = document.querySelector("#videoFsBtn")!; + videoFsBtn.addEventListener("click", () => { + videoCanvas.requestFullscreen(); }); let wakeLock: WakeLockSentinel; document.addEventListener("fullscreenchange", async () => { if (document.fullscreenElement) { - depthCanvas.width = fsWidth; - depthCanvas.height = fsHeight; + videoCanvas.width = fsWidth; + videoCanvas.height = fsHeight; try { wakeLock = await navigator.wakeLock.request(); } catch (e) { console.warn("wakeLock failed"); } } else { - depthCanvas.width = 640; - depthCanvas.height = 480; + videoCanvas.width = 640; + videoCanvas.height = 480; wakeLock?.release(); } }); + let reader: ReadableStreamDefaultReader; + let camStream: ReadableStream; const runStream = async () => { try { - depthStreamCb.checked = true; - const depthStream = await kinect.camera!.streamDepthFrames(); - for await (const frame of depthStream) { - const colorMarch = window.performance.now() / 10; - const grayFrame = KinectCamera.unpackDepthFrame(frame!.buffer); - - // frame is 11bit/u16gray, expand for canvas rgba - const rgbaFrame = new Uint8ClampedArray(640 * 480 * 4); - for (let i = 0; i < grayFrame.length; i++) { - const pixel16 = grayFrame[i]; - - // this counts as art - rgbaFrame[i * 4 + 0] = ((pixel16 << 1) + colorMarch) & 0xff; - rgbaFrame[i * 4 + 1] = ((pixel16 << 2) + colorMarch) & 0xff; - rgbaFrame[i * 4 + 2] = ((pixel16 << 3) + colorMarch) & 0xff; - - rgbaFrame[i * 4 + 3] = pixel16 < 2047 ? 0xff : 0; + await camera.ready; + switch (videoModeOption.value) { + case "depth": { + await camera.setMode( + modes( + { + ...DefaultModes.DEPTH, + flip: videoFlipCb.checked ? 1 : 0, + }, + DefaultModes.OFF, + ), + ); + + if (camera.depth.rawDeveloper) + camera.depth.rawDeveloper.customFn = customDepthRgba; + else console.error("failed to set custom deraw"); + camStream = camera.depth.readable as ReadableStream; + break; + } + case "visible": { + await camera.setMode( + modes(DefaultModes.OFF, { + ...DefaultModes.VISIBLE, + flip: videoFlipCb.checked ? 1 : 0, + }), + ); + camStream = camera.video.readable as ReadableStream; + break; } + case "ir": { + await camera.setMode( + modes(DefaultModes.OFF, { + ...DefaultModes.INFRARED, + flip: videoFlipCb.checked ? 1 : 0, + }), + ); + camStream = camera.video.readable as ReadableStream; + break; + } + default: + camStream = camera.video.readable as ReadableStream; + } - const drawFrame = new ImageData(rgbaFrame, 640, 480); + reader = camStream.getReader(); + console.log("got reader", reader); + const frameGenerator = async function* () { + while (true) { + const frame = await reader + .read() + .catch((e) => console.error("its ok lol", e)); + if (!frame || frame.done) break; + yield frame.value; + } + }; + for await (const drawFrame of frameGenerator()) { if (document.fullscreenElement) - depthCtx.putImageData(drawFrame, fsZeroX, fsZeroY); - else depthCtx.putImageData(drawFrame, 0, 0); + videoCanvas2dCtx.putImageData(drawFrame, fsZeroX, fsZeroY); + else videoCanvas2dCtx.putImageData(drawFrame, 0, 0); + frameCounter.push(true); } } catch (e) { cameraDemo.disabled = true; cameraDemo.classList.add("disabled"); + throw e; + } finally { + if (camStream.locked) reader.releaseLock(); } }; const endStream = async () => { - depthStreamCb.checked = false; - await kinect.camera?.endDepthStream(); - depthCtx.clearRect(0, 0, 640, 480); + if (camStream.locked) reader.releaseLock(); + await camera.setMode(modes(DefaultModes.OFF, DefaultModes.OFF)); }; - depthStreamCb.addEventListener("change", () => - depthStreamCb.checked ? runStream() : endStream(), - ); - runStream(); } -function setupMotorDemo(kinect: KinectDevice) { +function setupMotorDemo(motorDevice: USBDevice) { + motorDevice.open(); + const motor = new Motor(motorDevice); const motorDemo = document.querySelector("#motorDemo")!; motorDemo.hidden = false; motorDemo.disabled = false; @@ -216,7 +293,7 @@ function setupMotorDemo(kinect: KinectDevice) { const ledInput = document.querySelector("#ledInput")!; ledInput.addEventListener("click", () => { ledMode = (ledMode + 1) % 4; - kinect.motor?.cmdSetLed(ledMode); + motor?.setLed(ledMode); const [styleColor, nameColor] = ledModes[ledMode]; ledInput.textContent = `LED is ${nameColor}`; ledInput.style.background = styleColor; @@ -230,12 +307,12 @@ function setupMotorDemo(kinect: KinectDevice) { const tiltInput = document.querySelector("#tiltInput")!; tiltInput.addEventListener("change", () => { const angle = parseInt(tiltInput.value); - kinect.motor!.cmdSetTilt(angle); + motor!.setTilt(angle); }); setInterval(() => { - kinect - .motor!.cmdGetState() + motor! + .getState() .then( (motorState: { angle?: number; @@ -245,7 +322,7 @@ function setupMotorDemo(kinect: KinectDevice) { const { angle, servo, accel } = motorState; angleDisplay.textContent = String(angle); servoDisplay.textContent = String(servoModes[servo]); - accelDisplay.textContent = String(accel); //.join(", "); + accelDisplay.textContent = String(accel); }, ) .catch(() => { diff --git a/demo/src/style.css b/demo/src/style.css index eb6e795..8dc3f93 100644 --- a/demo/src/style.css +++ b/demo/src/style.css @@ -39,8 +39,13 @@ label { } #plugItIn img { - float: right; - width: 50%; + width: 100%; + max-width: 640px; +} + +#plugItIn video { + width: 100%; + max-width: 640px; } .disabled .canvasBack { diff --git a/demo/tsconfig.json b/demo/tsconfig.json index 82ce101..b1353ee 100644 --- a/demo/tsconfig.json +++ b/demo/tsconfig.json @@ -23,5 +23,5 @@ }, "include": [ "src" - ], + ] } \ No newline at end of file diff --git a/package.json b/package.json index 3be894c..00b2fc4 100644 --- a/package.json +++ b/package.json @@ -2,26 +2,35 @@ "name": "@webnect/webnect", "author": "turbocrime", "description": "a webUSB driver for the xbox360 kinect", - "version": "0.0.2", + "version": "0.0.3", "type": "module", "workspaces": ["demo"], - "exports": { ".": "./src/index.ts" }, + "exports": { + ".": "./src/index.ts", + "./util": "./src/util/index.ts", + "./stream": "./src/stream/index.ts" + }, "files": ["./dist", "./src"], "scripts": { "dev": "pnpm -C demo dev", "build": "tsc", "clean": "rm -rf ./dist/*", - "lint": "rome check src", + "lint": "biome check ./src", + "format": "biome format --write src", "prepublishOnly": "pnpm clean && pnpm lint && pnpm build" }, "publishConfig": { "access": "public", - "exports": { ".": "./dist/index.js" }, + "exports": { + ".": "./dist/index.js", + "./util": "./dist/util/index.js", + "./stream": "./dist/stream/index.js" + }, "types": "./dist/index.d.ts" }, "devDependencies": { "@types/w3c-web-usb": "^1.0.6", - "rome": "^12.1.3", - "typescript": "^5.1.6" + "@biomejs/biome": "1.0.0", + "typescript": "^5.2.2" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9f24aab..3551c9f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,4 +1,4 @@ -lockfileVersion: '6.0' +lockfileVersion: '6.1' settings: autoInstallPeers: true @@ -8,19 +8,19 @@ importers: .: devDependencies: + '@biomejs/biome': + specifier: 1.0.0 + version: 1.0.0 '@types/w3c-web-usb': specifier: ^1.0.6 version: 1.0.6 - rome: - specifier: ^12.1.3 - version: 12.1.3 typescript: - specifier: ^5.1.6 - version: 5.1.6 + specifier: ^5.2.2 + version: 5.2.2 demo: dependencies: - '@webnect/webnect': + '@webnect/webnect/': specifier: workspace:../ version: link:.. devDependencies: @@ -36,6 +36,74 @@ importers: packages: + /@biomejs/biome@1.0.0: + resolution: {integrity: sha512-Y5CND1QZ5pF6hc4dFw5ItDutv9KJO91ksLdBIFyvHL7LmXN0UomqyyRWryvrqq+YlA8Q58cR6sqjjQuMp9E2Ig==} + engines: {node: '>=14.*'} + hasBin: true + requiresBuild: true + optionalDependencies: + '@biomejs/cli-darwin-arm64': 1.0.0 + '@biomejs/cli-darwin-x64': 1.0.0 + '@biomejs/cli-linux-arm64': 1.0.0 + '@biomejs/cli-linux-x64': 1.0.0 + '@biomejs/cli-win32-arm64': 1.0.0 + '@biomejs/cli-win32-x64': 1.0.0 + dev: true + + /@biomejs/cli-darwin-arm64@1.0.0: + resolution: {integrity: sha512-3v7kEyxkf3D246esH+q/lDK5wWn+xLCXZpHCuc1itAmC35GkEc6S7um6C1VD3XKXLx6N0sJR/rTmjKiRGV32Ig==} + engines: {node: '>=14.*'} + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@biomejs/cli-darwin-x64@1.0.0: + resolution: {integrity: sha512-uxIMt/X7TQWicjsImkqMvUUEqaFZTOJJrtEhlHl/eIaETWJmK3uAR7ihIWctpGJnN16sUgpLgwczc7FETqu/PQ==} + engines: {node: '>=14.*'} + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@biomejs/cli-linux-arm64@1.0.0: + resolution: {integrity: sha512-kJWtu3Xr4MdHV2Yn4U+eZudAGPgv0kRCjWAyzLRewJiqE5TLPrX08imB9SU1n3+VxNO8e2JJ0tWWBHo4J+aSEg==} + engines: {node: '>=14.*'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@biomejs/cli-linux-x64@1.0.0: + resolution: {integrity: sha512-FK6hYZ0Lkk39eXYx1+2ZWtLkApc0RdOpcjDVM96JbvI0bxqvNnm193BPXuxh5A/fCl6N28RNUvcKnZ5LbgZ0Yw==} + engines: {node: '>=14.*'} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@biomejs/cli-win32-arm64@1.0.0: + resolution: {integrity: sha512-kE+OY2isEJHBodiLPMlybZckHkl3CQWsvXuJEvSxkoMhLbGDPEV3yZ/0lEph3BlxP3KP5vUO3hOFGaTvHFOuqQ==} + engines: {node: '>=14.*'} + cpu: [arm64] + os: [win32] + requiresBuild: true + dev: true + optional: true + + /@biomejs/cli-win32-x64@1.0.0: + resolution: {integrity: sha512-Ko6ZsbmbScPMEnh/xz4mwDSCZIUCAEjbbbnUVApgAAL2+1Hoe7Vnhh2RiwYRqy3tHrBIMDwXkSxj0vlf1G3EHg==} + engines: {node: '>=14.*'} + cpu: [x64] + os: [win32] + requiresBuild: true + dev: true + optional: true + /@esbuild/android-arm64@0.18.16: resolution: {integrity: sha512-wsCqSPqLz+6Ov+OM4EthU43DyYVVyfn15S4j1bJzylDpc1r1jZFFfJQNfDuT8SlgwuqpmpJXK4uPlHGw6ve7eA==} engines: {node: '>=12'} @@ -234,54 +302,6 @@ packages: dev: true optional: true - /@rometools/cli-darwin-arm64@12.1.3: - resolution: {integrity: sha512-AmFTUDYjBuEGQp/Wwps+2cqUr+qhR7gyXAUnkL5psCuNCz3807TrUq/ecOoct5MIavGJTH6R4aaSL6+f+VlBEg==} - cpu: [arm64] - os: [darwin] - requiresBuild: true - dev: true - optional: true - - /@rometools/cli-darwin-x64@12.1.3: - resolution: {integrity: sha512-k8MbWna8q4LRlb005N2X+JS1UQ+s3ZLBBvwk4fP8TBxlAJXUz17jLLu/Fi+7DTTEmMhM84TWj4FDKW+rNar28g==} - cpu: [x64] - os: [darwin] - requiresBuild: true - dev: true - optional: true - - /@rometools/cli-linux-arm64@12.1.3: - resolution: {integrity: sha512-X/uLhJ2/FNA3nu5TiyeNPqiD3OZoFfNfRvw6a3ut0jEREPvEn72NI7WPijH/gxSz55znfQ7UQ6iM4DZumUknJg==} - cpu: [arm64] - os: [linux] - requiresBuild: true - dev: true - optional: true - - /@rometools/cli-linux-x64@12.1.3: - resolution: {integrity: sha512-csP17q1eWiUXx9z6Jr/JJPibkplyKIwiWPYNzvPCGE8pHlKhwZj3YHRuu7Dm/4EOqx0XFIuqqWZUYm9bkIC8xg==} - cpu: [x64] - os: [linux] - requiresBuild: true - dev: true - optional: true - - /@rometools/cli-win32-arm64@12.1.3: - resolution: {integrity: sha512-RymHWeod57EBOJY4P636CgUwYA6BQdkQjh56XKk4pLEHO6X1bFyMet2XL7KlHw5qOTalzuzf5jJqUs+vf3jdXQ==} - cpu: [arm64] - os: [win32] - requiresBuild: true - dev: true - optional: true - - /@rometools/cli-win32-x64@12.1.3: - resolution: {integrity: sha512-yHSKYidqJMV9nADqg78GYA+cZ0hS1twANAjiFibQdXj9aGzD+s/IzIFEIi/U/OBLvWYg/SCw0QVozi2vTlKFDQ==} - cpu: [x64] - os: [win32] - requiresBuild: true - dev: true - optional: true - /@types/node@20.4.4: resolution: {integrity: sha512-CukZhumInROvLq3+b5gLev+vgpsIqC2D0deQr/yS1WnxvmYLlJXZpaQrQiseMY+6xusl79E04UjWoqyr+t1/Ew==} dev: true @@ -329,8 +349,8 @@ packages: '@esbuild/win32-x64': 0.18.16 dev: true - /fsevents@2.3.2: - resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + /fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] requiresBuild: true @@ -361,21 +381,7 @@ packages: engines: {node: '>=14.18.0', npm: '>=8.0.0'} hasBin: true optionalDependencies: - fsevents: 2.3.2 - dev: true - - /rome@12.1.3: - resolution: {integrity: sha512-e+ff72hxDpe/t5/Us7YRBVw3PBET7SeczTQNn6tvrWdrCaAw3qOukQQ+tDCkyFtS4yGsnhjrJbm43ctNbz27Yg==} - engines: {node: '>=14.*'} - hasBin: true - requiresBuild: true - optionalDependencies: - '@rometools/cli-darwin-arm64': 12.1.3 - '@rometools/cli-darwin-x64': 12.1.3 - '@rometools/cli-linux-arm64': 12.1.3 - '@rometools/cli-linux-x64': 12.1.3 - '@rometools/cli-win32-arm64': 12.1.3 - '@rometools/cli-win32-x64': 12.1.3 + fsevents: 2.3.3 dev: true /source-map-js@1.0.2: @@ -383,8 +389,8 @@ packages: engines: {node: '>=0.10.0'} dev: true - /typescript@5.1.6: - resolution: {integrity: sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==} + /typescript@5.2.2: + resolution: {integrity: sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==} engines: {node: '>=14.17'} hasBin: true dev: true @@ -422,5 +428,5 @@ packages: postcss: 8.4.27 rollup: 3.26.3 optionalDependencies: - fsevents: 2.3.2 + fsevents: 2.3.3 dev: true diff --git a/rome.json b/rome.json index d59ef33..61274a3 100644 --- a/rome.json +++ b/rome.json @@ -1,5 +1,5 @@ { - "$schema": "https://docs.rome.tools/schemas/12.1.3/schema.json", + "$schema": "https://biomejs.dev/schemas/1.0.0/schema.json", "organizeImports": { "enabled": false }, @@ -8,8 +8,7 @@ "rules": { "recommended": true, "style": { - "noNonNullAssertion": "off", - "useEnumInitializers": "warn" + "noNonNullAssertion": "off" }, "suspicious": { "noDebugger": "warn" diff --git a/src/CamCommand.ts b/src/CamCommand.ts new file mode 100644 index 0000000..7df6914 --- /dev/null +++ b/src/CamCommand.ts @@ -0,0 +1,176 @@ +import { CamUsbCommand, CamUsbControl, CamMagic } from "./enum/cam"; + +const CMD_HEADER_SIZE = 8; // bytes +const RESPONSE_TIMEOUT_MS = 200; +const RESPONSE_RETRY_MS = 15; + +export type CamCommandOut = CamCommand & { + magic: CamMagic.COMMAND_OUT; + response: Promise; +}; +export type CamCommandIn = CamCommand & { + magic: CamMagic.COMMAND_IN; + response: false; +}; + +export class CamCommand extends Uint16Array { + header: DataView; + + private _response?: Promise; + private resolve?: (value: CamCommandIn) => void; + // rome-ignore lint/suspicious/noExplicitAny: reject for any reason + private reject?: (reason?: any) => void; + + get magic() { + return this.header.getUint16(0, true); + } + get cmdLength() { + // size in i16 elements, sans header + return this.header.getUint16(2, true); + } + get cmdId() { + return this.header.getUint16(4, true); + } + get cmdTag() { + return this.header.getUint16(6, true); + } + get cmdContent() { + return new Uint16Array(this.buffer, CMD_HEADER_SIZE); + } + + set cmdTag(tag: number) { + this.header.setUint16(2, tag); + } + + set response(response: CamCommandIn | Error) { + if (response instanceof CamCommand) this.resolve!(response); + else this.reject!(response); + } + + get response(): Promise | false { + return this.magic === CamMagic.COMMAND_OUT && this._response!; + } + + constructor( + cmdBufferOrOpts: + | ArrayBuffer + | { cmdId: CamUsbCommand; tag: number; content: Uint16Array }, + ) { + const superBuffer = + cmdBufferOrOpts instanceof ArrayBuffer + ? cmdBufferOrOpts.slice( + 0, + CMD_HEADER_SIZE + + new DataView(cmdBufferOrOpts).getUint16(2, true) * 2, + ) + : new ArrayBuffer(CMD_HEADER_SIZE + cmdBufferOrOpts.content.byteLength); + + super(superBuffer); + + if (cmdBufferOrOpts instanceof ArrayBuffer) { + // no remaining buffer init + } else { + const { cmdId, tag, content } = cmdBufferOrOpts; + this.set([CamMagic.COMMAND_OUT, content.length, cmdId, tag]); + this.set(content, CMD_HEADER_SIZE / content.BYTES_PER_ELEMENT); + } + + this.header = new DataView(this.buffer, 0, CMD_HEADER_SIZE); + + if (this.magic === CamMagic.COMMAND_OUT) + this._response = new Promise((resolve, reject) => { + this.resolve = resolve; + this.reject = reject; + }); + } +} + +export class CamCommandIO { + private dev: USBDevice; + private pending: Map = new Map(); + + listening = false; + + constructor(dev: USBDevice) { + this.dev = dev; + } + + async transform( + chunk: CamCommandOut, + cont: TransformStreamDefaultController, + ) { + cont.enqueue(await this.sendCmd(chunk)); + } + + async pullResponse() { + this.listening = Boolean(this.pending.size); + if (!this.listening) return; + + const transfer = this.dev.controlTransferIn( + { + requestType: "vendor", + recipient: "device", + request: CamUsbControl.CAMERA, + value: 0, + index: 0, + }, + 512, // TODO: really? + ); + + await transfer.then((usbResult) => { + if (usbResult.status !== "ok") + return console.warn("Command response bad", usbResult); + if (!usbResult.data?.byteLength) + return console.warn("Command response empty"); + + let multiResponse = 0; + do { + const res = new CamCommand( + usbResult.data.buffer.slice(multiResponse), + ) as CamCommandIn; + const pend = this.pending.get(res.cmdTag); + if (pend) { + pend.response = res; + this.pending.delete(res.cmdTag); + } else console.warn("Command response unexpected", usbResult); + multiResponse += + CMD_HEADER_SIZE + res.cmdLength * res.BYTES_PER_ELEMENT; + } while (multiResponse < usbResult.data.byteLength); + }); + + // TODO: better rate limit + setTimeout(() => this.pullResponse(), RESPONSE_RETRY_MS); + } + + async sendCmd(cmd: CamCommandOut): Promise { + this.pending.set(cmd.cmdTag, cmd); + + const usbResult = await this.dev.controlTransferOut( + { + requestType: "vendor", + recipient: "device", + request: CamUsbControl.CAMERA, + value: 0, + index: 0, + }, + cmd.buffer, + ); + + if (usbResult.status !== "ok") + throw new Error(`Command failed ${usbResult}`); + + if (!this.listening) this.pullResponse(); + + return Promise.race([ + cmd.response, + new Promise((_, reject) => + setTimeout( + () => reject(new Error(`Command response timeout ${cmd.cmdTag}`)), + RESPONSE_TIMEOUT_MS, + ), + ), + ]).finally(() => { + this.pending.delete(cmd.cmdTag); + }); + } +} diff --git a/src/Camera.ts b/src/Camera.ts new file mode 100644 index 0000000..70f639b --- /dev/null +++ b/src/Camera.ts @@ -0,0 +1,236 @@ +import type { + CamIsoWorkerInitReply, + CamIsoWorkerActiveMsg, + CamIsoWorkerActiveReply, + CamIsoWorkerInitMsg, +} from "./worker/CamIsoWorker"; + +import type { CamModeSet } from "./util/mode"; + +import { CamStream, CamFrameDeveloper } from "./stream/CamStream"; + +import { + CamOption, + CamType, + CamUsbCommand, + CamIsoEndpoint, + OFF, +} from "./enum/cam"; + +import { parseModeOpts } from "./util/mode"; + +import { + CamCommand, + CamCommandOut, + CamCommandIn, + CamCommandIO, +} from "./CamCommand"; + +const getDeviceIndex = (d: USBDevice) => + navigator.usb.getDevices().then((ds) => ds.indexOf(d)); + +export class Camera { + dev: USBDevice; + + _registers: Record; + + [CamIsoEndpoint.DEPTH]: CamStream; + [CamIsoEndpoint.VIDEO]: CamStream; + + cmdTag: number; // TODO: maximum? + cmdIO: CamCommandIO; + + usbWorker: Worker; + + ready: Promise; + + constructor( + dev: USBDevice, + cameraModes?: CamModeSet, + deraw = [true, true] as [ + CamFrameDeveloper | boolean, + CamFrameDeveloper | boolean, + ], + ) { + this.cmdTag = 0; + this.cmdIO = new CamCommandIO(dev); + + this.dev = dev; + this._registers = {} as Record; + + this[CamIsoEndpoint.VIDEO] = new CamStream(deraw[0]); + this[CamIsoEndpoint.DEPTH] = new CamStream(deraw[1]); + + this.usbWorker = new Worker( + new URL("./worker/CamIsoWorker.ts", import.meta.url), + { + name: "webnect", + type: "module", + credentials: "omit", + } as WorkerOptions, + ); + this.ready = this.initWorker() + .then(() => + this.setMode(parseModeOpts({} as CamModeSet, true, cameraModes)), + ) + .then(() => this); + } + + get depth() { + return this[CamIsoEndpoint.DEPTH]; + } + + get video() { + return this[CamIsoEndpoint.VIDEO]; + } + + async initWorker() { + const initMsg = { + type: "init", + config: { + dev: await getDeviceIndex(this.dev), + }, + } as CamIsoWorkerInitMsg; + + const initReply = new Promise((resolve, reject) => { + const initReplyListener = (event: MessageEvent) => { + if (event.data?.type === "init") { + event.data.video.pipeTo(this[CamIsoEndpoint.VIDEO].writable); + event.data.depth.pipeTo(this[CamIsoEndpoint.DEPTH].writable); + resolve(event.data); + this.usbWorker.removeEventListener("message", initReplyListener); + } + }; + this.usbWorker.addEventListener("message", initReplyListener); + setTimeout(() => { + this.usbWorker.removeEventListener("message", initReplyListener); + reject(new Error("Worker init timeout")); + }, 1000); + }); + + this.usbWorker.postMessage(initMsg); + return initReply; + } + + activeWorker(setBoth?: "stop" | "go") { + const activeMsg = { + type: "active", + video: setBoth ?? this[CamIsoEndpoint.VIDEO].mode.stream ? "go" : "stop", + depth: setBoth ?? this[CamIsoEndpoint.DEPTH].mode.stream ? "go" : "stop", + } as CamIsoWorkerActiveMsg; + + const activeReply = new Promise( + (resolve, reject) => { + const activeReplyListener = (event: MessageEvent) => { + if (event.data?.type === "active") { + if ( + activeMsg.depth === event.data.depth && + activeMsg.video === event.data.video + ) + resolve(event.data); + else reject(event.data); + this.usbWorker.removeEventListener("message", activeReplyListener); + } + }; + this.usbWorker.addEventListener("message", activeReplyListener); + setTimeout(() => { + this.usbWorker.removeEventListener("message", activeReplyListener); + reject(new Error("Worker timeout")); + }, 1000); + }, + ); + + this.usbWorker.postMessage(activeMsg); + return activeReply; + } + + async setMode(modeOpt?: CamModeSet) { + const modes = parseModeOpts( + { + [CamIsoEndpoint.VIDEO]: this[CamIsoEndpoint.VIDEO].mode, + [CamIsoEndpoint.DEPTH]: this[CamIsoEndpoint.DEPTH].mode, + }, + false, + modeOpt, + ); + + await this.activeWorker("stop"); + this[CamIsoEndpoint.VIDEO].mode = modes[CamIsoEndpoint.VIDEO]; + this[CamIsoEndpoint.DEPTH].mode = modes[CamIsoEndpoint.DEPTH]; + await this.writeModeRegisters(); + await this.activeWorker(); + } + + async writeModeRegisters() { + await Promise.all([ + this.writeRegister(CamOption.PROJECTOR_CYCLE, OFF), + this.writeRegister(CamOption.DEPTH_TYPE, CamType.NONE), + this.writeRegister(CamOption.VIDEO_TYPE, CamType.NONE), + ]); + { + const { format, res, fps, flip, stream } = + this[CamIsoEndpoint.DEPTH].mode; + + this.writeRegister(CamOption.DEPTH_FMT, format); + this.writeRegister(CamOption.DEPTH_RES, res); + this.writeRegister(CamOption.DEPTH_FPS, fps); + this.writeRegister(CamOption.DEPTH_FLIP, flip); + await this.writeRegister(CamOption.DEPTH_TYPE, stream); + } + + { + const { format, res, fps, flip, stream } = + this[CamIsoEndpoint.VIDEO].mode; + + switch (stream) { + case CamType.VISIBLE: { + this.writeRegister(CamOption.VISIBLE_FMT, format); + this.writeRegister(CamOption.VISIBLE_RES, res); + this.writeRegister(CamOption.VISIBLE_FPS, fps); + this.writeRegister(CamOption.VISIBLE_FLIP, flip); + await this.writeRegister(CamOption.VIDEO_TYPE, stream).catch((e) => + console.error("Caught", e), + ); + break; + } + case CamType.INFRARED: { + this.writeRegister(CamOption.INFRARED_FMT, format); + this.writeRegister(CamOption.INFRARED_RES, res); + this.writeRegister(CamOption.INFRARED_FPS, fps); + this.writeRegister(CamOption.INFRARED_FLIP, flip); + await this.writeRegister(CamOption.VIDEO_TYPE, stream); + break; + } + } + } + } + + async command( + cmdId: CamUsbCommand, + content: Uint16Array, + ): Promise { + const tag = this.cmdTag++; + const cmd = new CamCommand({ cmdId, content, tag }) as CamCommandOut; + const rCmd = this.cmdIO.sendCmd(cmd) as Promise; + return (await rCmd).cmdContent; + } + + async writeRegister(register: CamOption, value: number) { + console.log("WRITING", CamOption[register], value); + const write = await this.command( + CamUsbCommand.WRITE_REGISTER, + new Uint16Array([register, value]), + ); + if (write && write.length === 1 && write[0] === 0) return write; + else throw Error(`bad write ${write}`); + } + + async readRegister(register: number) { + const read = await this.command( + CamUsbCommand.READ_REGISTER, + new Uint16Array([register]), + ); + if (read && read.length === 2) return read; + else throw Error(`bad read ${read}`); + } +} diff --git a/src/kinectMotor.ts b/src/Motor.ts similarity index 52% rename from src/kinectMotor.ts rename to src/Motor.ts index 3cb16c3..2eb95f9 100644 --- a/src/kinectMotor.ts +++ b/src/Motor.ts @@ -1,50 +1,30 @@ -export const MAX_TILT = 30; -export const ACCEL = 819; - -enum MotorUsbControl { - SET_LED = 0x06, - SET_TILT = 0x31, - GET_STATE = 0x32, -} - -export enum ServoMode { - IDLE = 0, - LIMIT = 1, - MOVING = 4, -} - -export enum LedMode { - OFF = 0, - GREEN = 1, - RED = 2, - AMBER = 3, - BLINK_GREEN = 4, - ALSO_BLINK_GREEN = 5, // same as 4? - BLINK_RED_AMBER = 6, -} +import { MotorUsbControl, MotorLed, MotorServoState } from "./enum/motor"; export type MotorState = { angle?: number; // raw, half-degrees - servo: ServoMode; + servo: MotorServoState; accel: [number, number, number]; }; -export class KinectMotor { +export const MOTOR_MAX_TILT = 30; +const MOTOR_STATE_SIZE = 10; +const GRAVITY = 9.80665; +const ACCEL = 819; + +export const accelToG = (x: number, y: number, z: number) => { + const ag = ACCEL * GRAVITY; + return [x / ag, y / ag, z / ag]; +}; +export class Motor { dev: USBDevice; state?: MotorState; - led?: LedMode; + led?: MotorLed; constructor(device: USBDevice) { this.dev = device; } - static accelToG(x: number, y: number, z: number) { - const ag = ACCEL * 9.80665; - return [x / ag, y / ag, z / ag]; - } - - async cmdGetState() { - const STATE_SIZE_BYTES = 10; + async getState() { const { data } = await this.dev.controlTransferIn( { requestType: "vendor", @@ -53,10 +33,10 @@ export class KinectMotor { value: 0, index: 0, }, - STATE_SIZE_BYTES, + MOTOR_STATE_SIZE, ); - // TODO: validate header at data[0] + // TODO: validate header const accel: [number, number, number] = [ data!.getInt16(2), @@ -74,18 +54,18 @@ export class KinectMotor { return this.state; } - async cmdSetTilt(angle: number) { - return await this.dev.controlTransferOut({ + setTilt(angle: number) { + return this.dev.controlTransferOut({ requestType: "vendor", recipient: "device", request: MotorUsbControl.SET_TILT, - value: angle % MAX_TILT, // crude limit + value: angle % MOTOR_MAX_TILT, // crude limit index: 0, }); } - async cmdSetLed(led: LedMode) { - return await this.dev.controlTransferOut({ + setLed(led: MotorLed) { + return this.dev.controlTransferOut({ requestType: "vendor", recipient: "device", request: MotorUsbControl.SET_LED, diff --git a/src/enum/cam.ts b/src/enum/cam.ts new file mode 100644 index 0000000..3e34c63 --- /dev/null +++ b/src/enum/cam.ts @@ -0,0 +1,126 @@ +export type OFF = 0; +export const OFF = 0 as OFF; +export type ON = 1; +export const ON = 1 as ON; + +export enum CamMagic { + COMMAND_OUT = 0x4d47, + COMMAND_IN = 0x4252, + ISOCHRONOUS_IN = 0x5242, +} + +export enum CamUsbControl { + CAMERA = 0x00, +} + +export enum CamUsbCommand { + READ_REGISTER = 0x02, + WRITE_REGISTER = 0x03, + ZEROPLANE = 0x04, + REGISTRATION = 0x16, + CMOS = 0x95, +} + +// hardware register addresses +export enum CamOption { + VIDEO_TYPE = 0x05, // VISIBLE | INFRARED + DEPTH_TYPE = 0x06, // DEPTH + + VISIBLE_FMT = 0x0c, + VISIBLE_RES = 0x0d, + VISIBLE_FPS = 0x0e, + + DEPTH_FMT = 0x12, + DEPTH_RES = 0x13, + DEPTH_FPS = 0x14, + + INFRARED_BRIGHTNESS = 0x15, + + DEPTH_FLIP = 0x17, + + INFRARED_FMT = 0x19, + INFRARED_RES = 0x1a, + INFRARED_FPS = 0x1b, + + VISIBLE_FLIP = 0x47, + INFRARED_FLIP = 0x48, + + PROJECTOR_CYCLE = 0x105, +} + +export enum CamFps { + F_15P = 15, + F_30P = 30, +} + +export enum CamFmtDepth { + D_11B = 0b11, + D_10B = 0b10, +} + +export enum CamFmtVisible { + BAYER_8B = 0x00, + YUV_16B = 0x05, +} + +export enum CamFmtInfrared { + IR_10B = 0x00, +} + +export enum CamType { + NONE = 0, + VISIBLE = 0b001, + DEPTH = 0b010, + INFRARED = 0b011, +} + +// actual device output varies, generally gets chopped to standard +export enum CamRes { + LOW = 0, // QVGA + MED = 1, // VGA + HIGH = 2, // SXGA +} + +/* +// TODO: support cmos +// another set of hardware register addresses on the camera cmos +export enum CamCMOSOption { + WHITEBALANCE_MANUAL = 1 << 15, + EXPOSURE_AUTO = 1 << 14, // important + DEFECT_CORRECTION = 1 << 13, + //UNKNOWN_12 = 1 << 12, + //UNKNOWN_11 = 1 << 11, + LENS_SHADING = 1 << 10, + //UNKNOWN_9 = 1 << 9, + //UNKNOWN_8 = 1 << 8, + ANTIFLICKER = 1 << 7, // important + //UNKNOWN_6 = 1 << 6, + //UNKNOWN_5 = 1 << 5, + COLOR_RAW = 1 << 4, // important + EXPOSURE_WEIGHTED = 1 << 3, + EXPOSURE_WINDOW = 1 << 2, + WHITEBALANCE_AUTO = 1 << 1, // important +} +*/ + +// endpoint reports support for a larger max, but always sends these. +// this size includes the 12-byte header. +export enum CamIsoPacketSize { + VIDEO = 1920, + DEPTH = 1760, +} + +// iso packets contain identifiers +export enum CamIsoPacketFlag { + VIDEO = 0b1000_0000, + DEPTH = 0b0111_0000, + START = 0b0000_0001, + MID = 0b0000_0010, + END = 0b0000_0101, +} + +// usb endpoint id, not an array index +export enum CamIsoEndpoint { + VIDEO = 0x01, + DEPTH = 0x02, +} diff --git a/src/enum/index.ts b/src/enum/index.ts new file mode 100644 index 0000000..22dca32 --- /dev/null +++ b/src/enum/index.ts @@ -0,0 +1,2 @@ +export * from "./cam"; +export * from "./motor"; diff --git a/src/enum/motor.ts b/src/enum/motor.ts new file mode 100644 index 0000000..5e8ce60 --- /dev/null +++ b/src/enum/motor.ts @@ -0,0 +1,21 @@ +export enum MotorUsbControl { + SET_LED = 0x06, + SET_TILT = 0x31, + GET_STATE = 0x32, +} + +export enum MotorServoState { + IDLE = 0, + LIMIT = 1, + MOVING = 4, +} + +export enum MotorLed { + OFF = 0, + GREEN = 1, + RED = 2, + AMBER = 3, + BLINK_GREEN = 4, + BLINK_GREEN_TOO = 5, + BLINK_RED_AMBER = 6, +} diff --git a/src/index.ts b/src/index.ts index c3f19c6..17301ed 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,49 @@ -export const usbSupport = typeof navigator?.usb?.getDevices === "function"; +const usbSupport = typeof navigator?.usb?.getDevices === "function"; if (!usbSupport) console.error("WebUSB supported not detected!"); -export { KinectDevice, KinectVendorId, KinectProductId } from "./kinectDevice"; -export { KinectMotor } from "./kinectMotor"; -export { KinectCamera } from "./kinectCamera"; +export enum VendorId { + MICROSOFT = 0x045e, +} + +export enum ProductId { + NUI_MOTOR = 0x02b0, + NUI_CAMERA = 0x02ae, + NUI_AUDIO = 0x02ad, +} + +export const claimNuiCamera = async (d?: USBDevice): Promise => { + const dev = + d || + (await navigator.usb.requestDevice({ + filters: [ + { + vendorId: VendorId.MICROSOFT, + productId: ProductId.NUI_CAMERA, + }, + ], + })); + await dev.open(); + await dev.reset(); + await dev.selectConfiguration(1); + return dev; +}; + +export const claimNuiMotor = async (d?: USBDevice): Promise => { + const dev = + d || + (await navigator.usb.requestDevice({ + filters: [ + { + vendorId: VendorId.MICROSOFT, + productId: ProductId.NUI_MOTOR, + }, + ], + })); + return dev; +}; + +export * from "./Motor"; +export * from "./Camera"; +export * from "./enum/index"; +export * from "./util/index"; +export * from "./stream/index"; diff --git a/src/kinectCamera.ts b/src/kinectCamera.ts deleted file mode 100644 index 272bb9a..0000000 --- a/src/kinectCamera.ts +++ /dev/null @@ -1,272 +0,0 @@ -import { KinectStream } from "./kinectStream"; - -const MAGIC_OUT = 0x4d47; -const MAGIC_IN = 0x4252; -const HDR_SIZE = 8; - -const OFF = 0; - -enum CamUsbControl { - ANY_COMMAND = 0x00, -} - -// usb endpoint id, not an array index -enum CamUsbEndpoint { - VIDEO = 0x01, - DEPTH = 0x02, -} - -enum CamUsbCommand { - READ_REGISTER = 0x02, - WRITE_REGISTER = 0x03, - ZEROPLANE = 0x04, - REGISTRATION = 0x16, - CMOS = 0x95, -} - -enum CamRegAddr { - PROJECTOR = 0x105, - - VIDEO_ACTIVE = 0x05, // CamModeActive VISIBLE | IR - DEPTH_ACTIVE = 0x06, // CamModeActive DEPTH - - VIDEO_MODE = 0x0c, - VIDEO_RES = 0x0d, - VIDEO_FPS = 0x0e, - - DEPTH_BPP = 0x12, // 0b11 11bit, 0b10 10bit - DEPTH_RES = 0x13, - DEPTH_FPS = 0x14, - IR_BRIGHTNESS = 0x15, - - IR_MODE = 0x19, - IR_RES = 0x1a, - IR_FPS = 0x1b, - - DEPTH_FLIP = 0x17, - VIDEO_FLIP = 0x47, - IR_FLIP = 0x48, -} - -enum CamModeFps { - F_15P = 15, - F_30P = 30, -} - -enum CamModeDepth { - D_11B = 0b11, - D_10B = 0b10, -} - -/* -enum CamBitflag { - WHITEBALANCE_MANUAL = 1 << 15, - EXPOSURE_AUTO = 1 << 14, // important - DEFECT_CORRECTION = 1 << 13, - - LENS_SHADING = 1 << 10, - - ANTIFLICKER = 1 << 7, // important - - COLOR_RAW = 1 << 4, // important - EXPOSURE_WEIGHTED = 1 << 3, - EXPOSURE_WINDOW = 1 << 2, - WHITEBALANCE_AUTO = 1 << 1, // important -} - -enum CamModeVisible { - BAYER = 0x00, - YUV = 0x05, -} - -enum CamModeIR { - IR = 0x00, // called "luminance"??? -} -*/ - -enum CamModeActive { - VISIBLE = 0b001, - DEPTH = 0b010, - IR = 0b100, -} - -// Some res/video combos are incompatible, actual output res may vary. -// For instance, MEDIUM is 640x488 for IR. -enum CamModeRes { - LOW = 0, // QVGA - 320x240 - MED = 1, // VGA - 640x480 - HIGH = 2, // SXGA - 1280x1024 -} - -type CamMode = { - fps: CamModeFps; - res: CamModeRes; - depth?: CamModeDepth; - //visible?: CamModeVisible; - //ir?: CamModeIR; -}; - -export class KinectCamera { - dev: USBDevice; - tag: number; - - //visibleStream?: KinectStream; - //irStream?: KinectStream; - depthStream?: KinectStream; - mode: CamMode; - - constructor(device: USBDevice) { - this.dev = device; - this.tag = 1; - this.mode = { - // arbitrary default - depth: CamModeDepth.D_11B, - fps: CamModeFps.F_30P, - res: CamModeRes.MED, - }; - } - - async endDepthStream() { - this.mode.depth = undefined; - await this.writeRegister(CamRegAddr.DEPTH_ACTIVE, OFF); - this.depthStream?.abort(); - } - - async initDepthStream() { - await this.writeRegister(CamRegAddr.PROJECTOR, OFF); // disable projector autocycle. TODO: why? position? - await this.writeRegister(CamRegAddr.DEPTH_ACTIVE, OFF); // in case it was on - - await this.writeDepthMode(); - - await this.writeRegister(CamRegAddr.DEPTH_ACTIVE, CamModeActive.DEPTH); - await this.writeRegister(CamRegAddr.DEPTH_FLIP, OFF); // disable depth hflip. TODO: position? - } - - async writeDepthMode() { - // TODO: more modes - this.mode.depth = CamModeDepth.D_11B; - await this.writeRegister(CamRegAddr.DEPTH_BPP, 0b11 as CamModeDepth); - await this.writeRegister(CamRegAddr.DEPTH_RES, CamModeRes.MED); - await this.writeRegister(CamRegAddr.DEPTH_FPS, 30 as CamModeFps); - } - - async streamDepthFrames() { - console.log("called streamDepthFrames"); - await this.initDepthStream(); - const endpoint = - this.dev.configuration!.interfaces[0].alternate.endpoints.find( - (e) => e.endpointNumber === CamUsbEndpoint.DEPTH, - )!; - this.depthStream = new KinectStream(this.dev, endpoint); - return await this.depthStream.stream(); - } - - static unpackDepthFrame(frame: ArrayBuffer) { - const src = new Uint8Array(frame); - const dest = new Uint16Array(640 * 480); - let window = 0; - let bits = 0; - let s = 0; - let d = 0; - while (s < src.length) { - while (bits < 11 && s < src.length) { - window = (window << 8) | src[s++]; - bits += 8; - } - if (bits < 11) break; - bits -= 11; - dest[d++] = window >> bits; - window &= (1 << bits) - 1; - } - return dest; - } - - async command(cmdId: CamUsbCommand, body: Uint16Array) { - const cmdBuffer = new ArrayBuffer(HDR_SIZE + body.byteLength); - if (cmdBuffer.byteLength > 1024) throw Error("command exceeds 1024 bytes"); - - const cmdHeader = new Uint16Array(cmdBuffer, 0, HDR_SIZE / 2); - cmdHeader.set([MAGIC_OUT, body.length, cmdId, this.tag]); - - const cmdBody = new Uint16Array( - cmdBuffer, - cmdHeader.byteLength, - body.length, - ); - cmdBody.set(body); - - const usbResponse = await this.dev.controlTransferOut( - { - requestType: "vendor", - recipient: "device", - request: CamUsbControl.ANY_COMMAND, - value: 0, - index: 0, - }, - cmdBuffer, - ); - - if (usbResponse.status !== "ok") - throw new Error(`command failed ${usbResponse}`); - - let cmdResponse: USBInTransferResult; - do { - cmdResponse = await this.dev.controlTransferIn( - { - requestType: "vendor", - recipient: "device", - request: CamUsbControl.ANY_COMMAND, - value: 0, - index: 0, - }, - 512, - ); - await new Promise((resolve) => setTimeout(resolve, 10)); - } while (!cmdResponse!.data?.byteLength); - - const rHeader = new Uint16Array(cmdResponse.data!.buffer, 0, HDR_SIZE / 2); - const rBody = new Uint16Array(cmdResponse.data!.buffer, HDR_SIZE); - - const [rMagic, rLength, rCmdId, rTag] = rHeader; - console.info( - "response ", - CamUsbCommand[rCmdId], - "seq", - rTag, - "body", - ...rBody, - ); - - if (rMagic !== MAGIC_IN) return console.error(`bad magic ${rMagic}`); - if (rLength !== rBody.length) - return console.error(`bad length ${rLength} expected ${rBody.length}`); - if (rCmdId !== cmdId) - return console.error(`bad command ${rCmdId} expected ${cmdId}`); - if (rTag !== this.tag) - return console.error(`bad tag ${rTag} expected ${this.tag}`); - - this.tag++; - return rBody; - } - - async writeRegister(register: CamRegAddr, value: number) { - console.log("writeRegister", CamRegAddr[register], value); - const write = await this.command( - CamUsbCommand.WRITE_REGISTER, - new Uint16Array([register, value]), - ); - if (write?.length !== 1 || write[0] !== 0) - return console.warn(`bad write ${write}`); - return write; - } - - async readRegister(register: number) { - console.log("readRegister", CamRegAddr[register]); - const read = await this.command( - CamUsbCommand.READ_REGISTER, - new Uint16Array([register]), - ); - if (read!.length !== 2) return console.warn(`bad read ${read}`); - return read; - } -} diff --git a/src/kinectDevice.ts b/src/kinectDevice.ts deleted file mode 100644 index 447fd02..0000000 --- a/src/kinectDevice.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { KinectMotor } from "./kinectMotor"; -import { KinectCamera } from "./kinectCamera"; -//import { KinectAudio } from "./kinectAudio"; - -export enum KinectVendorId { - MICROSOFT = 0x045e, -} - -export enum KinectProductId { - NUI_MOTOR = 0x02b0, - NUI_CAMERA = 0x02ae, - NUI_AUDIO = 0x02ad, -} - -export class KinectDevice { - camera?: KinectCamera; - motor?: KinectMotor; - audio?: undefined; //KinectAudio; - ready: Promise; - - constructor(devices?: { - camera?: USBDevice | boolean; - motor?: USBDevice | boolean; - audio?: USBDevice | boolean; - }) { - this.ready = this.init( - devices?.camera ?? true, - devices?.motor ?? false, - devices?.audio ?? false, - ); - } - - async init( - camera?: USBDevice | boolean, - motor?: USBDevice | boolean, - audio?: USBDevice | boolean, - ) { - const dPromises = Array(); - if (motor) - dPromises.push(this.claimNuiMotor(motor === true ? undefined : motor)); - if (camera) - dPromises.push(this.claimNuiCamera(camera === true ? undefined : camera)); - if (audio) dPromises.push(Promise.resolve(undefined)); - await Promise.allSettled(dPromises); - return this; - } - - async claimNuiCamera(select?: USBDevice): Promise { - const dev = - select || - (await navigator.usb.requestDevice({ - filters: [ - { - vendorId: KinectVendorId.MICROSOFT, - productId: KinectProductId.NUI_CAMERA, - }, - ], - })); - await dev.open(); - await dev.selectConfiguration(1); - await dev.claimInterface(0); - this.camera = new KinectCamera(dev); - return this.camera; - } - - async claimNuiMotor(select?: USBDevice): Promise { - const dev = - select || - (await navigator.usb.requestDevice({ - filters: [ - { - vendorId: KinectVendorId.MICROSOFT, - productId: KinectProductId.NUI_MOTOR, - }, - ], - })); - await dev.open(); - await dev.selectConfiguration(1); - await dev.claimInterface(0); - this.motor = new KinectMotor(dev); - return this.motor; - } -} diff --git a/src/kinectStream.ts b/src/kinectStream.ts deleted file mode 100644 index 3b6dcaa..0000000 --- a/src/kinectStream.ts +++ /dev/null @@ -1,170 +0,0 @@ -const PKT_MAGIC = 0x5242; -const HDR_SIZE = 12; - -enum KinectFrameSize { - // TODO: more modes - DEPTH_11B = 422400, // (640 * 480 * 11) / 8 -} - -// endpoint specifies a larger max, but iso transfer sends these. -// this size includes the header. -enum KinectPacketSize { - DEPTH = 1760, - VIDEO = 1920, -} - -enum KinectPacketType { - DEPTH = 0x70, - VIDEO = 0x80, - START = 0b001, - MID = 0b010, - END = 0b101, -} - -export class KinectStream { - dev: USBDevice; - endP: USBEndpoint; - abortController: AbortController; - - packetType: KinectPacketType; - frameSize: KinectFrameSize; - packetSize: KinectPacketSize; - packetLoss: number; - - sync: boolean; - seq: number; - stats: { - parsed: number; - valid: number; - used: number; - skipped: number; - }; - - constructor(dev: USBDevice, endP: USBEndpoint) { - this.dev = dev; - this.endP = endP; - this.abortController = new AbortController(); - this.packetType = KinectPacketType.DEPTH; - this.packetSize = KinectPacketSize.DEPTH; - this.frameSize = KinectFrameSize.DEPTH_11B; - this.sync = false; - this.packetLoss = 0; - this.seq = 0; - this.stats = { parsed: 0, valid: 0, used: 0, skipped: 0 }; - } - - // rome-ignore lint/suspicious/noExplicitAny: abort for any reason - abort(reason?: any) { - this.abortController.abort(reason); - } - - // rome-ignore lint/suspicious/noExplicitAny: desync for any reason - desync(...reasons: any) { - this.sync = false; - console.warn("desync", this.seq, ...reasons); - } - - async *stream() { - const process = (pkt: USBIsochronousInTransferPacket) => { - const BUF_SIZE = pkt.data!.buffer.byteLength; - const PKT_OFFSET = pkt.data!.byteOffset; - const PKT_SIZE = pkt.data!.byteLength; - - if (PKT_SIZE < HDR_SIZE) return {}; - if (BUF_SIZE < PKT_OFFSET + PKT_SIZE) return {}; - - const pH = pkt.data!.buffer.slice(PKT_OFFSET, PKT_OFFSET + HDR_SIZE); - const pB = pkt.data!.buffer.slice( - PKT_OFFSET + HDR_SIZE, - PKT_OFFSET + PKT_SIZE, - ); - - const pHeader = new DataView(pH); - const pBody = new DataView(pB); - - const pMagic = pHeader.getUint16(0); - const pType: KinectPacketType = pHeader.getUint8(3); - const pSeq = pHeader.getUint8(5); - const pSize: number = pHeader.getUint16(6); // includes header - //const pTime = pHeader.getUint32(8); - - if (pMagic !== PKT_MAGIC) return {}; - if (!(pType & this.packetType)) return {}; - if (pSize !== pBody.byteLength + HDR_SIZE) return {}; - - const startFrame = pType === (this.packetType | KinectPacketType.START); - const endFrame = pType === (this.packetType | KinectPacketType.END); - - if (startFrame) { - if (!this.sync) console.info("stream sync, seq", pSeq); - this.sync = true; - this.seq = pSeq; - } - - this.packetLoss = pSeq - this.seq; - if (this.sync) { - this.seq = pSeq; - if (Math.abs(this.packetLoss) > 3 && this.packetLoss !== 255) - this.desync( - "packet loss", - this.packetLoss, - KinectPacketType[pType - this.packetType], - ); - } - - if (!this.sync) return {}; - - return { - pBody, - pSeq, - startFrame, - endFrame, - }; - }; - - const frame = new Uint8Array(this.frameSize); - let frameIdx = 0; - while (!this.abortController.signal.aborted) { - const transfer = await this.dev.isochronousTransferIn( - this.endP.endpointNumber, - Array(512).fill(this.endP.packetSize), - // TODO: number requested is arbitrary, and it never actually reaches packetSize?? - ); - for (const pkt in transfer.packets) { - this.stats.parsed++; - this.seq = (this.seq + 1) % 256; - const { pBody, startFrame, endFrame } = process(transfer.packets[pkt]); - if (pBody) { - this.stats.valid++; - const remaining = frame.byteLength - frameIdx; - if (startFrame) frameIdx = 0; - else if (remaining < pBody.byteLength) { - this.desync("long frame", { - frameIdx, - remaining, - pBody: pBody.byteLength, - stats: this.stats, - }); - //yield frame.slice(0, frameIdx); - //debugger; - frameIdx = 0; - continue; - } - frame.set(new Uint8Array(pBody.buffer), frameIdx); - this.stats.used++; - frameIdx += pBody.byteLength; - if (endFrame && this.sync) { - if (frameIdx < this.frameSize) - this.desync("short frame", { - frameIdx, - remaining, - stats: this.stats, - }); - else yield frame.slice(0, frameIdx); - frameIdx = 0; - } - } - } - } - } -} diff --git a/src/stream/CamFrameAssembler.ts b/src/stream/CamFrameAssembler.ts new file mode 100644 index 0000000..422195d --- /dev/null +++ b/src/stream/CamFrameAssembler.ts @@ -0,0 +1,55 @@ +import type { CamIsoPacket } from "./CamIsoParser"; + +export class CamFrameAssembler implements Transformer { + private frameIdx = 0; + private sync = false; + + private _frameSize: number; + private frame: Uint8Array; + + constructor(frameSize?: number) { + this._frameSize = frameSize ?? 0; + this.frame = new Uint8Array(this.frameSize); + } + + transform( + { body, loss, startFrame, endFrame }: CamIsoPacket, + c: TransformStreamDefaultController, + ) { + if (!this.frameSize) return; + if (loss > this.frameSize - this.frameIdx) this.desync("lost frame"); + this.frameIdx += loss; + if (startFrame) this.resync(); + if (body.byteLength > this.frameSize - this.frameIdx) + this.desync("long frame"); + if (this.sync) { + this.frame.set(new Uint8Array(body), this.frameIdx); + this.frameIdx += body.byteLength; + } + if (endFrame) { + if (this.frameSize > this.frameIdx) this.desync("short frame"); + if (this.sync) c.enqueue(this.frame.buffer.slice(0, this.frameIdx)); + this.frameIdx = 0; + } + } + + private desync(reason: string) { + if (this.sync) console.warn("desync", reason); + this.sync = false; + } + + private resync() { + this.sync = true; + this.frameIdx = 0; + } + + set frameSize(frameSize: number) { + this._frameSize = frameSize; + this.frame = new Uint8Array(this.frameSize); + this.desync("frame resize"); + } + + get frameSize() { + return this._frameSize; + } +} diff --git a/src/stream/CamIsoParser.ts b/src/stream/CamIsoParser.ts new file mode 100644 index 0000000..2cc2fcf --- /dev/null +++ b/src/stream/CamIsoParser.ts @@ -0,0 +1,115 @@ +import { CamIsoPacketFlag, CamMagic } from "../enum/cam"; +import { SerializedUSBIsochronousInTransferResult } from "./UnderlyingIsochronousSource"; + +export type CamIsoPacket = { + stream: CamIsoPacketFlag; + startFrame: boolean; + endFrame: boolean; + loss: number; + header: CamIsoPacketHeader; + body: ArrayBuffer; +}; + +type CamIsoPacketHeader = { + pType: number; + pSeq: number; + pSize: number; + pTime: number; +}; + +const PKT_HEADER_SIZE = 12; + +export class CamIsoParser + implements + Transformer< + USBIsochronousInTransferResult | SerializedUSBIsochronousInTransferResult, + CamIsoPacket + > +{ + seq: number; + packetSize: number; + packetFlag: CamIsoPacketFlag; + + constructor(packetFlag: CamIsoPacketFlag, packetSize: number) { + this.seq = 0; + this.packetSize = packetSize; + this.packetFlag = packetFlag; + } + + transform( + chunk: + | USBIsochronousInTransferResult + | SerializedUSBIsochronousInTransferResult, + c: TransformStreamDefaultController, + ) { + if ("serialized" in chunk) { + // a serialized usb transfer result + const { packets, data } = chunk; + for (const p of packets) { + if (p.status !== "ok" || p.byteLength < PKT_HEADER_SIZE) continue; + const parsed = this.parsePacket( + new DataView(data, p.byteOffset, p.byteLength), + ); + if (parsed) c.enqueue(parsed); + } + } else if ("data" in chunk && "packets" in chunk) { + // a live usb transfer result + for (const p of chunk.packets) { + if (!p.data || p.status !== "ok" || p.data.byteLength < PKT_HEADER_SIZE) + continue; + const parsed = this.parsePacket(p.data); + if (parsed) c.enqueue(parsed); + } + } else throw new TypeError("unknown chunk"); + } + + parseHeader = (pkt: DataView): CamIsoPacketHeader | false => + CamMagic.ISOCHRONOUS_IN === pkt.getUint16(0) && { + // 2 + pType: pkt.getUint8(3), + // 4 + pSeq: pkt.getUint8(5), + pSize: pkt.getUint16(6), + pTime: pkt.getUint32(8), // TODO: wtf + }; + + parseType = (pType: number) => + // high bits identify stream + this.packetFlag === (pType & 0xf0) && { + stream: this.packetFlag, + // low bits indicate frame boundaries + startFrame: (pType & 0x0f) === CamIsoPacketFlag.START, + //midFrame: (pType & 0x0f) === CamIsoPacketFlag.MID, + endFrame: (pType & 0x0f) === CamIsoPacketFlag.END, + }; + + parsePacket = (pktView: DataView) => { + const header = this.parseHeader(pktView); + if (!header) return; // not for us + const { pType, pSeq, pSize } = header; + + if (pSize !== pktView.byteLength) + return console.error("bad packet length", pktView.byteLength, header); + + const packetType = this.parseType(pType); + if (!packetType) return; // not for us + if (packetType.startFrame) this.seq = pSeq; + + let seqDelta = pSeq - this.seq; + if (seqDelta < 0) seqDelta += 256; + const loss = + seqDelta > 1 ? (seqDelta - 1) * (this.packetSize - PKT_HEADER_SIZE) : 0; + + this.seq = pSeq; + + return { + ...packetType, + loss, + header: header, + body: pktView.buffer.slice( + pktView.byteOffset + PKT_HEADER_SIZE, + pktView.byteOffset + pSize, + ), + }; + }; +} diff --git a/src/stream/CamStream.ts b/src/stream/CamStream.ts new file mode 100644 index 0000000..d9fa341 --- /dev/null +++ b/src/stream/CamStream.ts @@ -0,0 +1,112 @@ +import type { CamMode } from "../util/mode"; +import { selectFrameSize, STREAM_OFF } from "../util/mode"; +import { CamFrameAssembler } from "./CamFrameAssembler"; +import { CamIsoPacket } from "./CamIsoParser"; + +//import { CamCanvas } from "../util/CamCanvas"; +import { selectFnToRgba } from "../index"; +import { RESOLUTIONS } from "../index"; + +export class CamStream + implements TransformStream +{ + private _mode: CamMode; + + private packetStream?: ReadableStream; + + frameAssembler: CamFrameAssembler; + private packetSink: WritableStream; + private rawStream: ReadableStream; + + rawDeveloper?: CamFrameDeveloper; + private frameSink?: WritableStream; + private imageStream?: ReadableStream; + + constructor( + //mode: CamMode, + deraw?: CamFrameDeveloper | boolean, + packets?: ReadableStream, + ) { + this._mode = STREAM_OFF as CamMode; + this.packetStream = packets; + + this.frameAssembler = new CamFrameAssembler(selectFrameSize(this._mode)); + + if (deraw == null || deraw === true) + this.rawDeveloper = new CamFrameDeveloper(this._mode); + else if (deraw) this.rawDeveloper = deraw; + + const { readable: rawStream, writable: packetSink } = new TransformStream( + this.frameAssembler, + ); + this.rawStream = rawStream; + this.packetSink = packetSink; + if (this.packetStream) this.packetStream.pipeTo(this.packetSink); + + if (this.rawDeveloper) { + const { readable: imageStream, writable: frameSink } = + new TransformStream(this.rawDeveloper); + this.imageStream = imageStream; + this.frameSink = frameSink; + this.rawStream.pipeTo(this.frameSink); + } + } + + get readable() { + return this.imageStream ?? this.rawStream; + } + + get writable() { + return this.packetSink; + } + + get mode() { + return this._mode; + } + + set mode(mode: CamMode) { + // TODO: pause like UnderlyingIsochronousSource? + this._mode = mode; + this.frameAssembler.frameSize = selectFrameSize(mode); + if (this.rawDeveloper) this.rawDeveloper.mode = mode; + } +} + +type ToRgba = (b: ArrayBuffer) => Uint8ClampedArray; +export class CamFrameDeveloper implements Transformer { + private _mode: CamMode; + private _customFn?: ToRgba; + private rawToRgba: ToRgba; + + frameWidth: number; + + constructor(mode: CamMode, customFn?: (r: ArrayBuffer) => Uint8ClampedArray) { + this._mode = mode; + this._customFn = customFn; + this.rawToRgba = customFn ?? selectFnToRgba(mode)!; + this.frameWidth = (RESOLUTIONS[mode.res] ?? [640, 480])[0]; + } + + get mode() { + return this._mode; + } + + set mode(newMode: CamMode) { + this._mode = newMode; + this.rawToRgba = this.customFn ?? selectFnToRgba(this._mode)!; + this.frameWidth = (RESOLUTIONS[newMode.res] ?? [640, 480])[0]; + } + + set customFn(newCustomFn: ToRgba) { + this._customFn = newCustomFn; + this.rawToRgba = this.customFn ?? selectFnToRgba(this._mode)!; + } + + get customFn(): ToRgba | undefined { + return this._customFn; + } + + transform(raw: ArrayBuffer, c: TransformStreamDefaultController) { + c.enqueue(new ImageData(this.rawToRgba(raw), this.frameWidth)); + } +} diff --git a/src/stream/UnderlyingIsochronousSource.ts b/src/stream/UnderlyingIsochronousSource.ts new file mode 100644 index 0000000..10d292a --- /dev/null +++ b/src/stream/UnderlyingIsochronousSource.ts @@ -0,0 +1,107 @@ +const DEFAULT_BATCH_SIZE = 256; +const DEFAULT_PULL_RATE_LIMIT = 5; +const DEFAULT_MAX_PENDING_TRANSFERS = 2; + +export type SerializedUSBIsochronousInTransferResult = { + readonly serialized: true; + readonly data: ArrayBuffer; + readonly packets: { + readonly byteOffset: number; + readonly byteLength: number; + readonly status: USBTransferStatus; + }[]; +}; + +export const serializeIso = ( + r: USBIsochronousInTransferResult, +): SerializedUSBIsochronousInTransferResult => ({ + serialized: true, + data: r.data!.buffer, + packets: r.packets.map((p) => ({ + byteOffset: p.data!.byteOffset, + byteLength: p.data!.byteLength, + status: p.status!, + })), +}); + +type UnderlyingIsochronousSourceOptions = { + batchSize?: number; + pullRateLimit?: number; + maxPendingTransfers?: number; +}; + +export class UnderlyingIsochronousSource + implements UnderlyingDefaultSource +{ + device: USBDevice; + pendingTransfers: number; + endpointNumber: number; + packetSize: number; + + batchSize: number; + maxPendingTransfers: number; + pullRateLimit: number; + + paused: Promise | false = false; + unpause = () => {}; + + constructor( + device: USBDevice, + endpointNumber: number, + packetSize: number, + extraOpts?: UnderlyingIsochronousSourceOptions, + ) { + this.pendingTransfers = 0; + + this.device = device; + this.endpointNumber = endpointNumber; + this.packetSize = packetSize; + + const { + batchSize = DEFAULT_BATCH_SIZE, + pullRateLimit = DEFAULT_PULL_RATE_LIMIT, + maxPendingTransfers = DEFAULT_MAX_PENDING_TRANSFERS, + } = extraOpts || {}; + + this.batchSize = batchSize; + this.pullRateLimit = pullRateLimit; + this.maxPendingTransfers = maxPendingTransfers; + } + + start(cont: ReadableStreamDefaultController) { + Array(this.maxPendingTransfers).forEach(() => this.pull(cont)); + } + + pull(cont: ReadableStreamDefaultController) { + if (this.paused) return this.paused; + if (this.pendingTransfers < this.maxPendingTransfers) { + this.pendingTransfers++; + this.device + .isochronousTransferIn( + this.endpointNumber, + Array(this.batchSize).fill(this.packetSize), + ) + .then((r) => cont.enqueue(serializeIso(r))) + .catch((e) => cont.error(e)) + .finally(() => this.pendingTransfers--); + } + // TODO: rate limit necessary? + return new Promise((r) => setTimeout(r, this.pullRateLimit)); + } + + cancel() { + this.device.close(); + } + + active(s: "stop" | "go") { + if (s === "stop") + this.paused = new Promise((resolve) => { + this.unpause = resolve; + }); + else { + this.paused = false; + this.unpause(); + } + return s; + } +} diff --git a/src/stream/index.ts b/src/stream/index.ts new file mode 100644 index 0000000..8dd100c --- /dev/null +++ b/src/stream/index.ts @@ -0,0 +1,4 @@ +export * from "./CamFrameAssembler"; +export * from "./CamIsoParser"; +export * from "./CamStream"; +export * from "./UnderlyingIsochronousSource"; diff --git a/src/util/frame.ts b/src/util/frame.ts new file mode 100644 index 0000000..3ee6821 --- /dev/null +++ b/src/util/frame.ts @@ -0,0 +1,305 @@ +import type { CamMode } from "./mode"; +import { + CamFmtDepth, + CamFmtInfrared, + CamFmtVisible, + CamRes, + CamType, +} from "../enum/cam"; + +export const unpackGray = (bitsPerPixel: number, packedBuffer: ArrayBuffer) => { + const packed = new Uint8Array(packedBuffer); + const unpacked = new Uint16Array(packed.byteLength / (bitsPerPixel / 8)); + let window = 0; + let bits = 0; + let pI = 0; + let uI = 0; + while (pI < packed.length) { + while (bits < bitsPerPixel && pI < packed.length) { + window = (window << 8) | packed[pI++]; + bits += 8; + } + if (bits < bitsPerPixel) break; + bits -= bitsPerPixel; + + unpacked[uI++] = window >> bits; + window &= (1 << bits) - 1; + } + return unpacked; +}; + +export const grayToRgba = (bitsPerPixel: number, grayBuffer: ArrayBuffer) => { + const gray = new Uint16Array(grayBuffer); + const rgba = new Uint8ClampedArray(gray.length * 4); + const maxPixel = (1 << bitsPerPixel) - 1; + const reduceDepth = bitsPerPixel - 8; + for (let gI = 0, rI = 0; gI < gray.length; gI++, rI += 4) { + const pixel = gray[gI]; + if (pixel === maxPixel) continue; // pixels init transparent + const reduced = pixel >> reduceDepth; + rgba[rI + 0] = reduced; + rgba[rI + 1] = reduced; + rgba[rI + 2] = reduced; + rgba[rI + 3] = 0xff; + } + return rgba; +}; + +export const unpackGrayToRgba = ( + bitsPerPixel: number, + packedBuffer: ArrayBuffer, +) => { + const packed = new Uint8Array(packedBuffer); + const rgba = new Uint8ClampedArray( + (packed.byteLength / (bitsPerPixel / 8)) * 4, + ); + const maxPixel = (1 << bitsPerPixel) - 1; + const reduceDepth = bitsPerPixel - 8; + let window = 0; + let bits = 0; + let pI = 0; + let rI = 0; + + while (pI < packed.length) { + while (bits < bitsPerPixel && pI < packed.length) { + window = (window << 8) | packed[pI++]; + bits += 8; + } + if (bits < bitsPerPixel) break; + bits -= bitsPerPixel; + + const pixel = window >> bits; + window &= (1 << bits) - 1; + + if (pixel !== maxPixel) { + const reduced = pixel >> reduceDepth; + rgba[rI++] = reduced; + rgba[rI++] = reduced; + rgba[rI++] = reduced; + rgba[rI++] = 0xff; + } else rI += 4; // pixels init transparent + } + + return rgba; +}; + +// like in libfreenect +const t_gamma = new Uint16Array(2048).map((_, i) => { + const v = i / 2048.0; + return Math.round(v ** 3 * 6 * 6 * 256); +}); + +export const grayToGamma = (bitsPerPixel: number, grayBuffer: ArrayBuffer) => { + const gray = new Uint16Array(grayBuffer); + const rgba = new Uint8ClampedArray(gray.length * 4); + const maxPixel = (1 << bitsPerPixel) - 1; + const adjustDepth = 11 - bitsPerPixel; + + for (let grayI = 0; grayI < gray.length; grayI++) { + const pixel = gray[grayI]; + if (pixel === maxPixel) continue; + + const red = grayI << 2; + const green = red + 1; + const blue = red + 2; + const alpha = red + 3; + + const gamma = t_gamma[pixel << adjustDepth]; + const high = gamma >> 8; + const low = gamma & 0xff; + + switch (high) { + case 0: + rgba[red] = 0xff; + rgba[green] = 0xff - low; + rgba[blue] = 0xff - low; + rgba[alpha] = 0xff; + break; + case 1: + rgba[red] = 0xff; + rgba[green] = low; + rgba[blue] = 0x00; + rgba[alpha] = 0xff; + break; + case 2: + rgba[red] = 0xff - low; + rgba[green] = 0xff; + rgba[blue] = 0x00; + rgba[alpha] = 0xff; + break; + case 3: + rgba[red] = 0x00; + rgba[green] = 0xff; + rgba[blue] = low; + rgba[alpha] = 0xff; + break; + case 4: + rgba[red] = 0x00; + rgba[green] = 0xff - low; + rgba[blue] = 0xff; + rgba[alpha] = 0xff; + break; + case 5: + rgba[red] = 0x00; + rgba[green] = 0x00; + rgba[blue] = 0xff - low; + rgba[alpha] = 0xff; + break; + default: + rgba[red] = 0xff; + rgba[green] = 0xff; + rgba[blue] = 0xff; + rgba[alpha] = 0xff; + break; + } + } + return rgba; +}; + +export const bayerToRgba = ( + width: number, + height: number, + bayerBuffer: ArrayBuffer, +) => { + const bayer = new Uint8Array(bayerBuffer); + const rgba = new Uint8ClampedArray(width * height * 4); + + let isEvenRow = true; + for ( + let bayerI = 0, rgbaI = 0, col = 0; + bayerI < bayer.length; + // rome-ignore lint/style/noCommaOperator: linter is broken + bayerI++, col++, rgbaI += 4 + ) { + const isEvenCol = !(col & 1); + if (col >= width) { + col = 0; + isEvenRow = !isEvenRow; + } + + const red = rgbaI; + const green = rgbaI + 1; + const blue = rgbaI + 2; + const alpha = rgbaI + 3; + + if (isEvenRow === isEvenCol) { + // green kernel + if (isEvenCol) { + rgba[red] = (bayer[bayerI - 1] + bayer[bayerI + 1]) >> 1; + rgba[green] = bayer[bayerI]; + rgba[blue] = (bayer[bayerI - width] + bayer[bayerI + width]) >> 1; + rgba[alpha] = 0xff; + } else { + rgba[red] = (bayer[bayerI - width] + bayer[bayerI + width]) >> 1; + rgba[green] = bayer[bayerI]; + rgba[blue] = (bayer[bayerI - 1] + bayer[bayerI + 1]) >> 1; + rgba[alpha] = 0xff; + } + } else if (isEvenRow) { + // red kernel + rgba[red] = bayer[bayerI]; + rgba[green] = + (bayer[bayerI - 1] + + bayer[bayerI + 1] + + bayer[bayerI - width] + + bayer[bayerI + width]) >> + 2; + rgba[blue] = + (bayer[bayerI - width - 1] + + bayer[bayerI - width + 1] + + bayer[bayerI + width - 1] + + bayer[bayerI + width + 1]) >> + 2; + rgba[alpha] = 0xff; + } else { + // blue kernel + rgba[red] = + (bayer[bayerI - width - 1] + + bayer[bayerI - width + 1] + + bayer[bayerI + width - 1] + + bayer[bayerI + width + 1]) >> + 2; + rgba[green] = + (bayer[bayerI - 1] + + bayer[bayerI + 1] + + bayer[bayerI - width] + + bayer[bayerI + width]) >> + 2; + rgba[blue] = bayer[bayerI]; + rgba[alpha] = 0xff; + } + } + return rgba; +}; + +export const uyvyToRgba = ( + width: number, + height: number, + uyvyBuffer: ArrayBuffer, +) => { + const yuvPixelToRgbaPixel = (y: number, u: number, v: number) => [ + y + 1.402 * (v - 128), + y - 0.344136 * (u - 128) - 0.714136 * (v - 128), + y + 1.772 * (u - 128), + 0xff, + ]; + const uyvy = new Uint8Array(uyvyBuffer); + const rgba = new Uint8ClampedArray(width * height * 4); + let rgbaI = 0; + for (let uyvyI = 0; uyvyI < uyvy.length; uyvyI += 4) { + const u = uyvy[uyvyI]; + const y1 = uyvy[uyvyI + 1]; + const v = uyvy[uyvyI + 2]; + const y2 = uyvy[uyvyI + 3]; + + rgba.set(yuvPixelToRgbaPixel(y1, u, v), rgbaI); + rgbaI += 4; + rgba.set(yuvPixelToRgbaPixel(y2, u, v), rgbaI); + rgbaI += 4; + } + + return rgba; +}; + +export const RESOLUTIONS = { + [CamRes.LOW]: [320, 240, 320 * 240], + [CamRes.MED]: [640, 480, 640 * 480], + [CamRes.HIGH]: [1280, 1024, 1280 * 1024], +}; + +export const selectFnToRgba = ( + mode: CamMode, +): ((f: ArrayBuffer) => Uint8ClampedArray) => { + const [width, height] = RESOLUTIONS[mode.res] ?? [640, 480]; + switch (mode.stream) { + case CamType.VISIBLE: + if (mode.format === CamFmtVisible.BAYER_8B) + return (f) => bayerToRgba(width, height, f); + else if (mode.format === CamFmtVisible.YUV_16B) + return (f) => uyvyToRgba(width, height, f); + break; + case CamType.DEPTH: + if (mode.format === CamFmtDepth.D_11B) + return (f) => grayToGamma(11, unpackGray(11, f)); + //return (f) => unpackGrayToRgba(11, f); + else if (mode.format === CamFmtDepth.D_10B) + return (f) => unpackGrayToRgba(10, f); + break; + case CamType.INFRARED: + if (mode.format === CamFmtInfrared.IR_10B) + return (f) => unpackGrayToRgba(10, f); + break; + } + return (f: ArrayBuffer) => { + console.error("untransformed buffer"); + return new Uint8ClampedArray(f); + }; +}; + +export const frameToImageData = (mode: CamMode) => { + const fn = selectFnToRgba(mode)!; + const [w] = RESOLUTIONS[mode.res as CamRes]; + return new TransformStream({ + transform: (chunk, c) => c.enqueue(new ImageData(fn(chunk), w)), + }); +}; diff --git a/src/util/index.ts b/src/util/index.ts new file mode 100644 index 0000000..a62a590 --- /dev/null +++ b/src/util/index.ts @@ -0,0 +1,2 @@ +export * from "./mode"; +export * from "./frame"; diff --git a/src/util/mode.ts b/src/util/mode.ts new file mode 100644 index 0000000..80fb406 --- /dev/null +++ b/src/util/mode.ts @@ -0,0 +1,146 @@ +import { + CamFps, + CamFmtDepth, + CamFmtInfrared, + CamFmtVisible, + CamRes, + CamType, + CamIsoEndpoint, + CamIsoPacketFlag, + CamIsoPacketSize, + OFF, + ON, +} from "../enum/cam"; + +export type CamMode = { + stream: CamType; + format: CamFmtDepth | CamFmtVisible | CamFmtInfrared; + res: CamRes; + fps: CamFps; + flip: ON | OFF; +}; + +export type CamModeSet = Record; + +// TODO: throw invalid modes +export const selectFrameSize = ({ + stream, + format, + res, +}: Pick) => { + const frameDimension = { + [CamRes.LOW]: 320 * 240, + [CamRes.MED]: 640 * 480, + [CamRes.HIGH]: 1280 * 1024, + }; + + const irFrameDimension = { + ...frameDimension, + [CamRes.MED]: 640 * 488, + // TODO: other wierd ones? + }; + + const bitsPerPixel = { + [(CamType.VISIBLE << 4) | CamFmtVisible.BAYER_8B]: 8, + [(CamType.VISIBLE << 4) | CamFmtVisible.YUV_16B]: 16, + [(CamType.DEPTH << 4) | CamFmtDepth.D_10B]: 10, + [(CamType.DEPTH << 4) | CamFmtDepth.D_11B]: 11, + [(CamType.INFRARED << 4) | CamFmtInfrared.IR_10B]: 10, + }; + + switch (stream) { + case CamType.VISIBLE: + return (frameDimension[res] * bitsPerPixel[(stream << 4) | format]) / 8; + case CamType.DEPTH: + return (frameDimension[res] * bitsPerPixel[(stream << 4) | format]) / 8; + case CamType.INFRARED: + return (irFrameDimension[res] * bitsPerPixel[(stream << 4) | format]) / 8; + case CamType.NONE: + return 0; + default: + throw new TypeError("Invalid stream type"); + } +}; + +export const selectPacketSize = (mode: CamMode) => + mode.stream === CamType.DEPTH + ? CamIsoPacketSize.DEPTH + : CamIsoPacketSize.VIDEO; + +export const selectPacketFlag = (mode: CamMode) => + mode.stream === CamType.DEPTH + ? CamIsoPacketFlag.DEPTH + : CamIsoPacketFlag.VIDEO; + +const DEFAULT_MODE_VISIBLE = { + stream: CamType.VISIBLE, + format: CamFmtVisible.BAYER_8B, + res: CamRes.MED, + flip: OFF, + fps: CamFps.F_30P, +}; + +const DEFAULT_MODE_INFRARED = { + stream: CamType.INFRARED, + format: CamFmtInfrared.IR_10B, + res: CamRes.MED, + flip: OFF, + fps: CamFps.F_30P, +}; + +const DEFAULT_MODE_DEPTH = { + stream: CamType.DEPTH, + format: CamFmtDepth.D_11B, + res: CamRes.MED, + flip: OFF, + fps: CamFps.F_30P, +}; + +export const STREAM_OFF = { stream: CamType.NONE } as CamMode; + +export const modes = ( + depthMode?: Partial, + videoMode?: Partial, +) => + ({ + [CamIsoEndpoint.DEPTH]: depthMode ?? STREAM_OFF, + [CamIsoEndpoint.VIDEO]: videoMode ?? STREAM_OFF, + }) as CamModeSet; + +export const DefaultModes = { + [CamType.VISIBLE]: DEFAULT_MODE_VISIBLE, + VISIBLE: DEFAULT_MODE_VISIBLE, + [CamType.INFRARED]: DEFAULT_MODE_INFRARED, + INFRARED: DEFAULT_MODE_INFRARED, + [CamType.DEPTH]: DEFAULT_MODE_DEPTH, + DEPTH: DEFAULT_MODE_DEPTH, + [CamType.NONE]: STREAM_OFF, + NONE: STREAM_OFF, + OFF: STREAM_OFF, +}; + +export const parseModeOpts = ( + existing: CamModeSet, + useDefaults = false as typeof DefaultModes | boolean, + modeOpt = {} as Record>, +): Record => { + const defaults = useDefaults === true ? DefaultModes : useDefaults; + + const getUpdatedMode = ( + endpoint: CamIsoEndpoint, + mode?: Partial, + ): Partial => ({ + [endpoint]: { + ...(defaults && mode?.stream + ? defaults[mode.stream] + : existing[endpoint]), + ...mode, + }, + }); + + const fullMode = { + ...getUpdatedMode(CamIsoEndpoint.VIDEO, modeOpt[CamIsoEndpoint.VIDEO]), + ...getUpdatedMode(CamIsoEndpoint.DEPTH, modeOpt[CamIsoEndpoint.DEPTH]), + } as Record; + return fullMode; +}; diff --git a/src/util/tsconfig.json b/src/util/tsconfig.json new file mode 100644 index 0000000..6d696a0 --- /dev/null +++ b/src/util/tsconfig.json @@ -0,0 +1,8 @@ +{ + "compilerOptions": { + "lib": [ + "es2022", + "dom" + ] + } +} \ No newline at end of file diff --git a/src/worker/CamIsoWorker.ts b/src/worker/CamIsoWorker.ts new file mode 100644 index 0000000..975a13e --- /dev/null +++ b/src/worker/CamIsoWorker.ts @@ -0,0 +1,134 @@ +import type { CamIsoPacket } from "../stream/CamIsoParser"; + +declare const navigator: Navigator & { + usb: USB; +}; + +declare const self: Worker & { + postMessage: (msg: CamIsoWorkerReply, transfer?: Transferable[]) => void; +}; + +import { + CamIsoEndpoint, + CamIsoPacketFlag, + CamIsoPacketSize, +} from "../enum/cam"; +import { CamIsoParser } from "../stream/CamIsoParser"; +import { UnderlyingIsochronousSource } from "../stream/UnderlyingIsochronousSource"; + +export type CamIsoWorkerOpts = { + dev: number; + batchSize?: number; + devconf?: number; + iface?: number; + altiface?: number; +}; + +export type CamIsoWorkerInitMsg = { + type: "init"; + config: CamIsoWorkerOpts; +}; + +export type CamIsoWorkerActiveMsg = { + type: "active"; + depth: "stop" | "go"; + video: "stop" | "go"; +}; + +export type CamIsoWorkerMsg = CamIsoWorkerInitMsg | CamIsoWorkerActiveMsg; +export type CamIsoWorkerReply = CamIsoWorkerInitReply | CamIsoWorkerActiveReply; + +export type CamIsoWorkerInitReply = { + type: "init"; + depth: ReadableStream; + video: ReadableStream; +}; + +export type CamIsoWorkerActiveReply = CamIsoWorkerActiveMsg; + +const DEFAULT_USB_DEV = 0; +const DEFAULT_USB_IFACE = 0; + +let dev: USBDevice; +let iface: USBInterface; +//let devconf: USBConfiguration; +//let altiface: USBAlternateInterface; +let batchSize: number | undefined; + +const sources = { + [CamIsoEndpoint.DEPTH]: {} as UnderlyingIsochronousSource, + [CamIsoEndpoint.VIDEO]: {} as UnderlyingIsochronousSource, +}; + +self.addEventListener("message", async (event: { data: CamIsoWorkerMsg }) => { + switch (event.data?.type) { + case "init": { + await initUsb(event.data.config); + const depth = initStream("DEPTH"); + const video = initStream("VIDEO"); + self.postMessage( + { type: "init", depth, video } as CamIsoWorkerInitReply, + [depth, video] as Transferable[], + ); + break; + } + case "active": { + const { depth, video } = event.data; + sources[CamIsoEndpoint.DEPTH].active(depth); + sources[CamIsoEndpoint.VIDEO].active(video); + self.postMessage({ + type: "active", + depth, + video, + } as CamIsoWorkerActiveReply); + break; + } + default: { + console.error("Unknown message", event); + throw TypeError("Unknown message"); + } + } +}); + +const initUsb = async (opt: CamIsoWorkerOpts) => { + opt.dev ??= DEFAULT_USB_DEV; + opt.iface ??= DEFAULT_USB_IFACE; + + const d = await navigator.usb.getDevices(); + dev = d[opt.dev]; + + await dev.open(); + + if (opt.devconf != null) await dev.selectConfiguration(opt.devconf); + //devconf = dev.configuration!; + + iface = dev.configuration?.interfaces.find( + ({ interfaceNumber }) => interfaceNumber === opt.iface, + )!; + await dev.claimInterface(iface.interfaceNumber); + + if (opt.altiface != null) + await dev.selectAlternateInterface(iface.interfaceNumber, opt.altiface); + //altiface = iface.alternate; + + batchSize = opt.batchSize; +}; + +const initStream = (streamType: "DEPTH" | "VIDEO") => { + const source = new UnderlyingIsochronousSource( + dev, + CamIsoEndpoint[streamType], + CamIsoPacketSize[streamType], + { batchSize }, + ); + sources[CamIsoEndpoint[streamType]] = source; + const packetStream = new ReadableStream(source).pipeThrough( + new TransformStream( + new CamIsoParser( + CamIsoPacketFlag[streamType], + CamIsoPacketSize[streamType], + ), + ), + ); + return packetStream; +}; diff --git a/src/worker/tsconfig.json b/src/worker/tsconfig.json new file mode 100644 index 0000000..1ccb8db --- /dev/null +++ b/src/worker/tsconfig.json @@ -0,0 +1,8 @@ +{ + "compilerOptions": { + "lib": [ + "webworker", + "es2022" + ] + } +} \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index 364be88..4b7d602 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,15 +4,13 @@ "target": "es2022", "lib": [ "dom", - "es2022", + "es2022" ], "strict": true, "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "outDir": "dist", - "declaration": true, - "declarationMap": true, - "sourceMap": true, + "declaration": true }, "include": [ "src"