added displayLine command

extensions/src/doodlebot/LineDetection.ts

@@ -0,0 +1,61 @@
+import { createCanvas, loadImage } from 'canvas';
+import { endpoint, port } from "./enums";
+
+export class LineDetector {
+  private lastDetectedLine: number[][] = [];
+  private isProcessing = false;
+  private canvas: any;
+  private ctx: any;
+
+  constructor(private raspberryPiIp: string, private width = 640, private height = 480) {
+    this.canvas = createCanvas(this.width, this.height);
+    this.ctx = this.canvas.getContext('2d');
+  }
+
+  async detectLine(): Promise<number[][]> {
+    if (this.isProcessing) return this.lastDetectedLine;
+    this.isProcessing = true;
+
+    try {
+      const image = await loadImage(`http://${this.raspberryPiIp}:${port.camera}/${endpoint.video}`);
+      this.ctx.drawImage(image, 0, 0, this.width, this.height);
+      const imageData = this.ctx.getImageData(0, 0, this.width, this.height);
+      const lineCoordinates = this.processImageData(imageData);
+
+      if (lineCoordinates.length > 0) {
+        this.lastDetectedLine = lineCoordinates;
+      }
+
+      return this.lastDetectedLine;
+    } catch (error) {
+      console.error('Error detecting line:', error);
+      return this.lastDetectedLine;
+    } finally {
+      this.isProcessing = false;
+    }
+  }
+
+  private processImageData(imageData: ImageData): number[][] {
+    const lineCoordinates: number[][] = [];
+    const threshold = 50; 
+
+    for (let y = 0; y < this.height; y++) {
+      for (let x = 0; x < this.width; x++) {
+        const index = (y * this.width + x) * 4;
+        const r = imageData.data[index];
+        const g = imageData.data[index + 1];
+        const b = imageData.data[index + 2];
+
+        if (r < threshold && g < threshold && b < threshold) {
+          lineCoordinates.push([x, y]);
+        }
+      }
+    }
+    return lineCoordinates.sort((a, b) => a[1] - b[1]);
+  }
+}
+
+export function createLineDetector(raspberryPiIp: string): () => Promise<number[][]> {
+  const detector = new LineDetector(raspberryPiIp);
+  return () => detector.detectLine();
+}

extensions/src/doodlebot/index.ts

@@ -4,7 +4,7 @@ import Doodlebot from "./Doodlebot";
 import { splitArgsString } from "./utils";
 import EventEmitter from "events";
 import { categoryByGesture, classes, emojiByGesture, gestureDetection, gestureMenuItems, gestures, objectDetection } from "./detection";
-
+import { createLineDetector } from "./LineDetection";
 const details: ExtensionMenuDisplayDetails = {
   name: "Doodlebot",
   description: "Program a doodlebot robot",
@@ -38,7 +38,7 @@ const looper = (action: () => Promise<any>, profileMarker?: string) => {
 export default class DoodlebotBlocks extends extension(details, "ui", "indicators", "video", "drawable") {
   doodlebot: Doodlebot;
   private indicator: Promise<{ close(): void; }>;
-
+  private lineDetector: (() => Promise<number[][]>) | null = null;
   bluetoothEmitter = new EventEmitter();
 
   gestureLoop: ReturnType<typeof looper>;
@@ -95,6 +95,55 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator
     return drawable;
   }
 
+  @block({
+    type: "command",
+    text: "displayLine"
+  })
+  async displayLine() {
+    if (!this.lineDetector) {
+      const ipAddress = await this.doodlebot?.getIPAddress();
+      if (!ipAddress) {
+        console.error("Unable to get IP address for line detection");
+        return;
+      }
+      this.lineDetector = createLineDetector(ipAddress);
+    }
+
+    const lineCoordinates = await this.lineDetector();
+    if (lineCoordinates.length === 0) {
+      console.log("No line detected");
+      return;
+    }
+
+    // Create a drawable if it doesn't exist
+    if (!this.videoDrawable) {
+      this.videoDrawable = await this.createVideoStreamDrawable();
+    }
+
+    // Create a temporary canvas to draw on
+    const canvas = document.createElement('canvas');
+    canvas.width = this.imageStream.width; // Assume these properties exist
+    canvas.height = this.imageStream.height;
+    const ctx = canvas.getContext('2d');
+
+    if (ctx) {
+      ctx.drawImage(this.imageStream, 0, 0, canvas.width, canvas.height);
+
+      // Draw the line on the canvas
+      ctx.beginPath();
+      ctx.moveTo(lineCoordinates[0][0], lineCoordinates[0][1]);
+      for (let i = 1; i < lineCoordinates.length; i++) {
+        ctx.lineTo(lineCoordinates[i][0], lineCoordinates[i][1]);
+      }
+      ctx.strokeStyle = 'red';
+      ctx.lineWidth = 2;
+      ctx.stroke();
+
+      // Update the drawable with the new canvas content
+      this.videoDrawable.update(canvas);
+    }
+  }
+
   @buttonBlock("Connect Robot")
   connect() {
     this.openUI("Connect");