From 33a294612a2a375a2eaa41822293e7597d4abf9a Mon Sep 17 00:00:00 2001 From: "Github Action (authored by pmalacho-mit)" Date: Thu, 21 Nov 2024 13:39:25 -0800 Subject: [PATCH] ensuring extension and linedetector use the same stream --- extensions/src/doodlebot/LineDetection.ts | 57 ++++++++++++----------- extensions/src/doodlebot/index.ts | 8 +++- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/extensions/src/doodlebot/LineDetection.ts b/extensions/src/doodlebot/LineDetection.ts index 5d1de4186..16ecac9b7 100644 --- a/extensions/src/doodlebot/LineDetection.ts +++ b/extensions/src/doodlebot/LineDetection.ts @@ -24,27 +24,28 @@ export class LineDetector { private readonly MIN_PROCESS_INTERVAL = 100; private imageStream: HTMLImageElement | null = null; - constructor(private raspberryPiIp: string, width = 640, height = 480) { + constructor(private raspberryPiIp: string, imageStream: HTMLImageElement, width = 640, height = 480) { debug.info('Initializing LineDetector', { raspberryPiIp, width, height }); - + this.width = width; this.height = height; this.canvas = document.createElement('canvas'); this.canvas.width = width; this.canvas.height = height; - + const ctx = this.canvas.getContext('2d', { willReadFrequently: true }); if (!ctx) { debug.error('Failed to get canvas context'); throw new Error('Failed to get canvas context'); } - + this.ctx = ctx; this.lastDetectedLine = []; this.frameCount = 0; this.allCoordinates = []; - + debug.info('LineDetector initialized successfully'); + this.imageStream = imageStream; } async initialize(doodlebot: Doodlebot): Promise { @@ -63,40 +64,40 @@ export class LineDetector { private processImageData(imageData: ImageData): number[][] { debug.time('processImageData'); - + const lineCoordinates: number[][] = []; const maxY = Math.min(this.height, 400); const data = imageData.data; const width = this.width; - + for (let y = 0; y < maxY; y += 2) { const rowOffset = y * width * 4; for (let x = 0; x < width; x += 2) { const index = rowOffset + (x * 4); - if (data[index] < this.threshold && - data[index + 1] < this.threshold && - data[index + 2] < this.threshold) { + if (data[index] < this.threshold && + data[index + 1] < this.threshold && + data[index + 2] < this.threshold) { lineCoordinates.push([x, y]); } } } - + debug.info(`Found ${lineCoordinates.length} line points`); debug.timeEnd('processImageData'); - - return lineCoordinates.length > 0 + + return lineCoordinates.length > 0 ? lineCoordinates.sort((a, b) => a[1] - b[1]) : []; } async detectLine(doodlebot: Doodlebot, retries: number = 3): Promise { const now = Date.now(); - + if (this.isProcessing) { debug.warn('Detection already in progress, returning last result'); return this.lastDetectedLine; } - + if ((now - this.lastProcessTime) < this.MIN_PROCESS_INTERVAL) { debug.warn('Called too soon after last detection, returning last result'); return this.lastDetectedLine; @@ -115,14 +116,14 @@ export class LineDetector { while (attempt < retries) { try { debug.info(`Detection attempt ${attempt + 1}/${retries}`); - + debug.info('Clearing canvas and drawing new image'); this.ctx.clearRect(0, 0, this.width, this.height); this.ctx.drawImage(this.imageStream!, 0, 0, this.width, this.height); - + debug.info('Getting image data from canvas'); const imageData = this.ctx.getImageData(0, 0, this.width, this.height); - + debug.info('Processing image data'); const lineCoordinates = this.processImageData(imageData); @@ -132,16 +133,16 @@ export class LineDetector { firstPoint: lineCoordinates[0], lastPoint: lineCoordinates[lineCoordinates.length - 1] }); - + this.lastDetectedLine = lineCoordinates; - + if (this.collectLine) { this.allCoordinates.push(lineCoordinates); this.frameCount++; this.collectLine = false; debug.info('Line collected, frame count:', this.frameCount); } - + this.lastProcessTime = now; this.isProcessing = false; debug.timeEnd('detectLine'); @@ -153,18 +154,18 @@ export class LineDetector { const backoffTime = 100 * Math.pow(2, attempt); debug.info(`Waiting ${backoffTime}ms before next attempt`); await new Promise(resolve => setTimeout(resolve, backoffTime)); - + } catch (error) { debug.error(`Processing attempt ${attempt + 1} failed:`, error); attempt++; - + if (attempt === retries) { debug.warn('Max retries reached, returning last known good result'); this.isProcessing = false; debug.timeEnd('detectLine'); return this.lastDetectedLine; } - + const backoffTime = 100 * Math.pow(2, attempt); debug.info(`Waiting ${backoffTime}ms before retry`); await new Promise(resolve => setTimeout(resolve, backoffTime)); @@ -189,8 +190,8 @@ export class LineDetector { } getLastDetectedLine(): number[][] { - debug.info('Returning last detected line', { - points: this.lastDetectedLine.length + debug.info('Returning last detected line', { + points: this.lastDetectedLine.length }); return this.lastDetectedLine; } @@ -212,8 +213,8 @@ export class LineDetector { } } -export async function createLineDetector(raspberryPiIp: string): Promise<(doodlebot: Doodlebot) => Promise> { +export async function createLineDetector(raspberryPiIp: string, imageStream: HTMLImageElement): Promise<(doodlebot: Doodlebot) => Promise> { debug.info('Creating new LineDetector instance'); - const detector = new LineDetector(raspberryPiIp); + const detector = new LineDetector(raspberryPiIp, imageStream); return (doodlebot: Doodlebot) => detector.detectLine(doodlebot); } \ No newline at end of file diff --git a/extensions/src/doodlebot/index.ts b/extensions/src/doodlebot/index.ts index f4c535c37..93c2d0d39 100644 --- a/extensions/src/doodlebot/index.ts +++ b/extensions/src/doodlebot/index.ts @@ -87,6 +87,11 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator this.openUI("ReattachBLE"); } + async getImageStream() { + this.imageStream ??= await this.doodlebot?.getImageStream(); + return this.imageStream; + } + async createVideoStreamDrawable() { this.imageStream ??= await this.doodlebot?.getImageStream(); if (!this.imageStream) { @@ -118,7 +123,8 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator console.error("Unable to get IP address for line detection"); return; } - this.lineDetector = createLineDetector(ipAddress); + const imageStream = await this.getImageStream(); + this.lineDetector = createLineDetector(ipAddress, imageStream); } const lineCoordinates = await this.lineDetector();