Skip to content

Commit

Permalink
fixed integration
Browse files Browse the repository at this point in the history
  • Loading branch information
Brandon Lei authored and Brandon Lei committed Nov 14, 2024
1 parent a73a892 commit f67bd02
Show file tree
Hide file tree
Showing 3 changed files with 286 additions and 73 deletions.
187 changes: 155 additions & 32 deletions extensions/src/doodlebot/LineDetection.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,81 +12,204 @@ export class LineDetector {
private width = 640,
private height = 480,
private threshold = 70 // Threshold for detecting dark pixels
) {}
) {
console.log(`LineDetector initialized with IP: ${raspberryPiIp}`);
console.log(`Parameters: width=${width}, height=${height}, threshold=${threshold}`);
}

async detectLine(): Promise<number[][]> {
if (this.isProcessing) return this.lastDetectedLine;
if (this.isProcessing) {
console.log("Already processing, returning last detected line");
return this.lastDetectedLine;
}

console.log("Starting line detection...");
this.isProcessing = true;

try {
// Get image from endpoint
const response = await axios.get(
`http://${this.raspberryPiIp}:${port.camera}/${endpoint.video}`,
{ responseType: 'arraybuffer' }
);

// Convert response to Uint8Array for pixel processing
const buffer = Buffer.from(response.data);
const pixels = new Uint8Array(buffer);
const url = `http://${this.raspberryPiIp}:${port.camera}/${endpoint.video}`;
console.log(`Fetching DEBUG image from: ${url}`);

// Process the image data to find dark pixels
const lineCoordinates = this.processImageData(pixels);
const controller = new AbortController();
const timeout = setTimeout(() => {
controller.abort();
console.log("Fetch request timed out");
}, 5000);

if (lineCoordinates.length > 0) {
this.lastDetectedLine = lineCoordinates;
}
try {
const response = await fetch(url, {
method: 'GET',
mode: 'cors',
headers: {
'Accept': 'multipart/x-mixed-replace; boundary=frame'
},
signal: controller.signal
});

// Store coordinates for the first 7 frames (matching HTML version)
if (this.frameCount < 7) {
this.allCoordinates.push(lineCoordinates);
this.frameCount++;
if (this.frameCount === 7) {
this.logCoordinates();
clearTimeout(timeout);

if (!response.ok) {
console.error(`HTTP error! status: ${response.status}`);
throw new Error(`HTTP error! status: ${response.status}`);
}
}

return this.lastDetectedLine;
console.log("Response received, getting reader...");
const reader = response.body?.getReader();
if (!reader) {
throw new Error('No reader available');
}

console.log("Reading stream...");
const { value, done } = await reader.read();

if (done || !value) {
throw new Error('Stream ended unexpectedly');
}

console.log(`Received chunk of size: ${value.length}`);
// Debug the first few bytes to see what we're getting
console.log("First 20 bytes:", Array.from(value.slice(0, 20)).map(b => b.toString(16).padStart(2, '0')).join(' '));

// Look for the multipart boundary
const data = new TextDecoder().decode(value);
console.log("First 100 chars of data:", data.substring(0, 100));

// Try to find the content-type header
const contentTypeMatch = data.match(/Content-Type: ([^\r\n]+)/i);
if (contentTypeMatch) {
console.log("Content-Type found:", contentTypeMatch[1]);
}

// Try to find the actual image data after headers
const headerEndIndex = data.indexOf('\r\n\r\n');
if (headerEndIndex !== -1) {
console.log("Headers found, image data starts at:", headerEndIndex + 4);
const blobData = value.slice(headerEndIndex + 4);

// Create blob from image data
const blob = new Blob([blobData], { type: 'image/jpeg' });
console.log("Created blob of size:", blob.size);

// Convert to image
const img = new Image();
const imageUrl = URL.createObjectURL(blob);

await new Promise((resolve, reject) => {
img.onload = resolve;
img.onerror = (e) => {
console.error("Image load error:", e);
reject(e);
};
img.src = imageUrl;
});

// Clean up the stream
reader.cancel();

console.log("Image loaded, dimensions:", img.width, "x", img.height);

// Create canvas to get pixel data
const canvas = document.createElement('canvas');
canvas.width = img.width;
canvas.height = img.height;
const ctx = canvas.getContext('2d');

if (!ctx) {
throw new Error('Could not get canvas context');
}

// Draw image to canvas
ctx.drawImage(img, 0, 0);

// Get pixel data
const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
console.log("Got image data, processing pixels...");

// Process the image data
const lineCoordinates = this.processImageData(imageData.data);

// Clean up
URL.revokeObjectURL(imageUrl);

if (lineCoordinates.length > 0) {
this.lastDetectedLine = lineCoordinates;
}

return this.lastDetectedLine;
} else {
console.error("Could not find end of headers in response");
return this.lastDetectedLine;
}

} catch (fetchError) {
console.error("Fetch error:", fetchError);
if (fetchError.name === 'AbortError') {
console.log("Request was aborted due to timeout");
}
throw fetchError;
}
} catch (error) {
console.error('Error detecting line:', error);
return this.lastDetectedLine;
} finally {
this.isProcessing = false;
console.log("Line detection process completed");
}
}

private processImageData(pixels: Uint8Array): number[][] {
private processImageData(pixels: Uint8ClampedArray): number[][] {
const lineCoordinates: number[][] = [];
const maxY = Math.min(this.height, 400);

console.log(`Processing image data: ${this.width}x${maxY}`);
console.log(`Pixel array length: ${pixels.length}, Expected length: ${this.width * this.height * 3}`);

// Process only up to y < 400 (matching HTML version)
for (let y = 0; y < Math.min(this.height, 400); y++) {
// Add sample pixel values logging
const samplePixels: {[key: string]: number[]} = {};

for (let y = 0; y < maxY; y++) {
for (let x = 0; x < this.width; x++) {
const index = (y * this.width + x) * 3; // RGB format
const r = pixels[index];
const g = pixels[index + 1];
const b = pixels[index + 2];

// Log a few sample pixels
if (y % 100 === 0 && x % 100 === 0) {
samplePixels[`${x},${y}`] = [r, g, b];
}

// Check if pixel is dark (below threshold)
if (r < this.threshold && g < this.threshold && b < this.threshold) {
lineCoordinates.push([x, y]);
}
}
}

console.log("Sample pixel values:", samplePixels);
console.log(`Found ${lineCoordinates.length} dark pixels`);

// Sort coordinates by y-value (top to bottom)
return lineCoordinates.sort((a, b) => a[1] - b[1]);
const sortedCoordinates = lineCoordinates.sort((a, b) => a[1] - b[1]);
console.log("First few coordinates:", sortedCoordinates.slice(0, 5));

return sortedCoordinates;
}

private logCoordinates(): void {
// Log coordinates for debugging (similar to file writing in HTML version)
console.log('Collected coordinates from 7 frames:');
console.log('=== Collected coordinates from 7 frames ===');
this.allCoordinates.forEach((frame, index) => {
console.log(`Frame ${index + 1}:`);
console.log(frame.map(coord => coord.join(',')).join('\n'));
console.log(`Frame ${index + 1}: ${frame.length} points`);
if (frame.length > 0) {
console.log('First 3 coordinates:', frame.slice(0, 3));
console.log('Last 3 coordinates:', frame.slice(-3));
}
});
}
}

export function createLineDetector(raspberryPiIp: string): () => Promise<number[][]> {
console.log("Creating new LineDetector instance");
const detector = new LineDetector(raspberryPiIp);
return () => detector.detectLine();
}
90 changes: 49 additions & 41 deletions extensions/src/doodlebot/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { categoryByGesture, classes, emojiByGesture, gestureDetection, gestureMe
//import { createLineDetector } from "./LineDetection";
import { line0, line1, line2, line3, line4, line5, line6, line7, line8 } from './Points';
import { followLine } from "./LineFollowing";
import { createLineDetector } from "./LineDetection";

const details: ExtensionMenuDisplayDetails = {
name: "Doodlebot",
Expand Down Expand Up @@ -87,6 +88,11 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator

async createVideoStreamDrawable() {
this.imageStream ??= await this.doodlebot?.getImageStream();
if (!this.imageStream) {
console.error("Failed to get image stream");
return;
}
console.log("Image stream dimensions:", this.imageStream.width, "x", this.imageStream.height);
const drawable = this.createDrawable(this.imageStream);
drawable.setVisible(true);
const self = this;
Expand All @@ -103,47 +109,49 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator
text: "displayLine"
})
async displayLine() {
// console.log("displayLine");
// if (!this.lineDetector) {
// const ipAddress = await this.doodlebot?.getIPAddress();
// if (!ipAddress) {
// console.error("Unable to get IP address for line detection");
// return;
// }
// this.lineDetector = createLineDetector(ipAddress);
// }

// const lineCoordinates = await this.lineDetector();
// if (lineCoordinates.length === 0) {
// console.log("No line detected");
// return;
// }

// console.log("Line coordinates:", JSON.stringify(lineCoordinates));

// if (!this.videoDrawable) {
// this.videoDrawable = await this.createVideoStreamDrawable();
// }

// const canvas = document.createElement('canvas');
// canvas.width = this.imageStream.width; // Assume these properties exist
// canvas.height = this.imageStream.height;
// const ctx = canvas.getContext('2d');

// if (ctx) {
// ctx.drawImage(this.imageStream, 0, 0, canvas.width, canvas.height);

// ctx.beginPath();
// ctx.moveTo(lineCoordinates[0][0], lineCoordinates[0][1]);
// for (let i = 1; i < lineCoordinates.length; i++) {
// ctx.lineTo(lineCoordinates[i][0], lineCoordinates[i][1]);
// }
// ctx.strokeStyle = 'red';
// ctx.lineWidth = 2;
// ctx.stroke();

// this.videoDrawable.update(canvas);
// }
console.log("displayLine");
if (!this.lineDetector) {
const ipAddress = await this.doodlebot?.getIPAddress();
console.log("DEBUG IP Address:", ipAddress);
if (!ipAddress) {
console.error("Unable to get IP address for line detection");
return;
}
this.lineDetector = createLineDetector(ipAddress);
}

const lineCoordinates = await this.lineDetector();
console.log("Raw line coordinates:", lineCoordinates);
if (lineCoordinates.length === 0) {
console.log("No line detected");
return;
}

console.log("Line coordinates:", JSON.stringify(lineCoordinates));

if (!this.videoDrawable) {
this.videoDrawable = await this.createVideoStreamDrawable();
}

const canvas = document.createElement('canvas');
canvas.width = this.imageStream.width; // Assume these properties exist
canvas.height = this.imageStream.height;
const ctx = canvas.getContext('2d');

if (ctx) {
ctx.drawImage(this.imageStream, 0, 0, canvas.width, canvas.height);

ctx.beginPath();
ctx.moveTo(lineCoordinates[0][0], lineCoordinates[0][1]);
for (let i = 1; i < lineCoordinates.length; i++) {
ctx.lineTo(lineCoordinates[i][0], lineCoordinates[i][1]);
}
ctx.strokeStyle = 'red';
ctx.lineWidth = 2;
ctx.stroke();

this.videoDrawable.update(canvas);
}
}

@buttonBlock("Connect Robot")
Expand Down
Loading

0 comments on commit f67bd02

Please sign in to comment.