Merge remote-tracking branch 'origin/line_detection_2' into line_comm…

…ands
mitmedialab · Oct 4, 2024 · 5dd3257 · 5dd3257
2 parents 1eff7cb + 29851ea
commit 5dd3257
Show file tree

Hide file tree

Showing 3 changed files with 254 additions and 2 deletions.
diff --git a/LineDetection.html b/LineDetection.html
@@ -0,0 +1,143 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Line Detection</title>
+    <style>
+        body { font-family: Arial, sans-serif; text-align: center; }
+        #outputImage { max-width: 100%; height: auto; border: 1px solid #ddd; }
+        #controls { margin: 20px 0; }
+        #ipInput { padding: 5px; width: 200px; }
+        button { padding: 5px 10px; cursor: pointer; }
+    </style>
+    <script>
+        const port = { camera: 8000 };
+        const endpoint = { video: 'video_feed' };
+
+        class LineDetector {
+            constructor(raspberryPiIp, width = 640, height = 480) {
+                this.raspberryPiIp = raspberryPiIp = "192.168.41.214";
+                this.width = width;
+                this.height = height;
+                this.canvas = document.createElement('canvas');
+                this.canvas.width = width;
+                this.canvas.height = height;
+                this.ctx = this.canvas.getContext('2d');
+                this.lastDetectedLine = [];
+                this.frameCount = 0;
+                this.allCoordinates = [];
+            }
+
+            async detectLine() {
+                try {
+                    const image = await this.loadImage(`http://${this.raspberryPiIp}:${port.camera}/${endpoint.video}`);
+                    this.ctx.drawImage(image, 0, 0, this.width, this.height);
+                    const imageData = this.ctx.getImageData(0, 0, this.width, this.height);
+                    const lineCoordinates = this.processImageData(imageData);
+                    console.log("coordinates");
+                    console.log(lineCoordinates);
+                    if (lineCoordinates.length > 0) {
+                        this.lastDetectedLine = lineCoordinates;
+                    }
+
+                    if (this.frameCount < 7) {
+                        this.allCoordinates.push(lineCoordinates);
+                        this.frameCount++;
+                        if (this.frameCount === 7) {
+                            this.writeCoordinatesToFile(this.allCoordinates);
+                        }
+                    }
+
+                    this.drawLine(this.lastDetectedLine);
+                    return this.canvas.toDataURL();
+                } catch (error) {
+                    console.error('Error detecting line:', error);
+                    return null;
+                }
+            }
+
+            loadImage(src) {
+                return new Promise((resolve, reject) => {
+                    const img = new Image();
+                    img.crossOrigin = "Anonymous";
+                    img.onload = () => resolve(img);
+                    img.onerror = reject;
+                    img.src = src;
+                });
+            }
+
+            processImageData(imageData) {
+                const lineCoordinates = [];
+                const threshold = 70;
+
+                for (let y = 0; y < this.height; y++) {
+                    for (let x = 0; x < this.width; x++) {
+                        const index = (y * this.width + x) * 4;
+                        const [r, g, b] = imageData.data.slice(index, index + 3);
+
+                        if (r < threshold && g < threshold && b < threshold && y < 400) {
+                            lineCoordinates.push([x, y]);
+                        }   
+                    }
+                }
+                return lineCoordinates.sort((a, b) => a[1] - b[1]);
+            }
+
+            drawLine(coordinates) {
+                this.ctx.beginPath();
+                this.ctx.strokeStyle = 'blue';
+                this.ctx.lineWidth = 2;
+                coordinates.forEach(([x, y], i) => {
+                this.ctx.arc(x, y, 5, 0, Math.PI*2);
+                });
+                this.ctx.stroke();
+            }
+
+            writeCoordinatesToFile(allCoordinates) {
+                const content = allCoordinates.map((frame, index) => 
+                    `Frame ${index + 1}:\n${frame.map(coord => coord.join(',')).join('\n')}\n`
+                ).join('\n');
+                const blob = new Blob([content], { type: 'text/plain' });
+                const a = document.createElement('a');
+                a.href = URL.createObjectURL(blob);
+                a.download = 'coordinates.txt';
+                a.click();
+                URL.revokeObjectURL(a.href);
+            }
+        }
+
+        let detector;
+        let updateInterval;
+
+        function initializeDetector() {
+            const raspberryPiIp = document.getElementById('ipInput').value;
+            if (!raspberryPiIp) {
+                alert('Please enter a valid IP address');
+                return;
+            }
+            detector = new LineDetector(raspberryPiIp);
+            clearInterval(updateInterval);
+            updateImage();
+            updateInterval = setInterval(updateImage, 500);
+        }
+
+        async function updateImage() {
+            if (detector) {
+                const imageDataUrl = await detector.detectLine();
+                if (imageDataUrl) {
+                    document.getElementById('outputImage').src = imageDataUrl;
+                }
+            }
+        }
+    </script>
+</head>
+<body>
+    <h1>Line Detection</h1>
+    <div id="controls">
+        <input type="text" id="ipInput" placeholder="IP Address">
+        <button onclick="initializeDetector()">Start Detection</button>
+    </div>
+    <img id="outputImage" alt="Processed Image">
+</body>
+</html>
diff --git a/extensions/src/doodlebot/LineDetection.ts b/extensions/src/doodlebot/LineDetection.ts
@@ -0,0 +1,61 @@
+import { createCanvas, loadImage } from 'canvas';
+import { endpoint, port } from "./enums";
+
+export class LineDetector {
+  private lastDetectedLine: number[][] = [];
+  private isProcessing = false;
+  private canvas: any;
+  private ctx: any;
+
+  constructor(private raspberryPiIp: string, private width = 640, private height = 480) {
+    this.canvas = createCanvas(this.width, this.height);
+    this.ctx = this.canvas.getContext('2d');
+  }
+
+  async detectLine(): Promise<number[][]> {
+    if (this.isProcessing) return this.lastDetectedLine;
+    this.isProcessing = true;
+
+    try {
+      const image = await loadImage(`http://${this.raspberryPiIp}:${port.camera}/${endpoint.video}`);
+      this.ctx.drawImage(image, 0, 0, this.width, this.height);
+      const imageData = this.ctx.getImageData(0, 0, this.width, this.height);
+      const lineCoordinates = this.processImageData(imageData);
+
+      if (lineCoordinates.length > 0) {
+        this.lastDetectedLine = lineCoordinates;
+      }
+
+      return this.lastDetectedLine;
+    } catch (error) {
+      console.error('Error detecting line:', error);
+      return this.lastDetectedLine;
+    } finally {
+      this.isProcessing = false;
+    }
+  }
+
+  private processImageData(imageData: ImageData): number[][] {
+    const lineCoordinates: number[][] = [];
+    const threshold = 50; 
+
+    for (let y = 0; y < this.height; y++) {
+      for (let x = 0; x < this.width; x++) {
+        const index = (y * this.width + x) * 4;
+        const r = imageData.data[index];
+        const g = imageData.data[index + 1];
+        const b = imageData.data[index + 2];
+
+        if (r < threshold && g < threshold && b < threshold) {
+          lineCoordinates.push([x, y]);
+        }
+      }
+    }
+    return lineCoordinates.sort((a, b) => a[1] - b[1]);
+  }
+}
+
+export function createLineDetector(raspberryPiIp: string): () => Promise<number[][]> {
+  const detector = new LineDetector(raspberryPiIp);
+  return () => detector.detectLine();
+}
diff --git a/extensions/src/doodlebot/index.ts b/extensions/src/doodlebot/index.ts
@@ -4,7 +4,7 @@ import Doodlebot from "./Doodlebot";
 import { splitArgsString } from "./utils";
 import EventEmitter from "events";
 import { categoryByGesture, classes, emojiByGesture, gestureDetection, gestureMenuItems, gestures, objectDetection } from "./detection";
-
+import { createLineDetector } from "./LineDetection";
 const details: ExtensionMenuDisplayDetails = {
   name: "Doodlebot",
   description: "Program a doodlebot robot",
@@ -38,7 +38,7 @@ const looper = (action: () => Promise<any>, profileMarker?: string) => {
 export default class DoodlebotBlocks extends extension(details, "ui", "indicators", "video", "drawable") {
   doodlebot: Doodlebot;
   private indicator: Promise<{ close(): void; }>;
-
+  private lineDetector: (() => Promise<number[][]>) | null = null;
   bluetoothEmitter = new EventEmitter();
 
   gestureLoop: ReturnType<typeof looper>;
@@ -95,6 +95,54 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator
     return drawable;
   }
 
+  @block({
+    type: "command",
+    text: "displayLine"
+  })
+  async displayLine() {
+    console.log("displayLine");
+    if (!this.lineDetector) {
+      const ipAddress = await this.doodlebot?.getIPAddress();
+      if (!ipAddress) {
+        console.error("Unable to get IP address for line detection");
+        return;
+      }
+      this.lineDetector = createLineDetector(ipAddress);
+    }
+
+    const lineCoordinates = await this.lineDetector();
+    if (lineCoordinates.length === 0) {
+      console.log("No line detected");
+      return;
+    }
+
+    console.log("Line coordinates:", JSON.stringify(lineCoordinates));
+
+    if (!this.videoDrawable) {
+      this.videoDrawable = await this.createVideoStreamDrawable();
+    }
+
+    const canvas = document.createElement('canvas');
+    canvas.width = this.imageStream.width; // Assume these properties exist
+    canvas.height = this.imageStream.height;
+    const ctx = canvas.getContext('2d');
+
+    if (ctx) {
+      ctx.drawImage(this.imageStream, 0, 0, canvas.width, canvas.height);
+
+      ctx.beginPath();
+      ctx.moveTo(lineCoordinates[0][0], lineCoordinates[0][1]);
+      for (let i = 1; i < lineCoordinates.length; i++) {
+        ctx.lineTo(lineCoordinates[i][0], lineCoordinates[i][1]);
+      }
+      ctx.strokeStyle = 'red';
+      ctx.lineWidth = 2;
+      ctx.stroke();
+
+      this.videoDrawable.update(canvas);
+    }
+  }
+
   @buttonBlock("Connect Robot")
   connect() {
     this.openUI("Connect");