diff --git a/extensions/src/doodlebot/ArrayError.svelte b/extensions/src/doodlebot/ArrayError.svelte
new file mode 100644
index 000000000..497616f7c
--- /dev/null
+++ b/extensions/src/doodlebot/ArrayError.svelte
@@ -0,0 +1,40 @@
+
+
+
+
Cannot load Doodlebot's sound/image server!
+ Please try reloading the page.
+
+
+
diff --git a/extensions/src/doodlebot/CustomArgument.svelte b/extensions/src/doodlebot/CustomArgument.svelte
new file mode 100644
index 000000000..f2dbd9189
--- /dev/null
+++ b/extensions/src/doodlebot/CustomArgument.svelte
@@ -0,0 +1,83 @@
+
+
+
+ {#each soundFiles as f}
+ {#if value == f}
+
+ {/if}
+ {#if value != f}
+
+ {/if}
+ {/each}
+
+
+
diff --git a/extensions/src/doodlebot/Doodlebot.ts b/extensions/src/doodlebot/Doodlebot.ts
index 33de6ed72..111f207c8 100644
--- a/extensions/src/doodlebot/Doodlebot.ts
+++ b/extensions/src/doodlebot/Doodlebot.ts
@@ -147,6 +147,90 @@ export default class Doodlebot {
private isStopped = true; // should this be initializeed more intelligently?
+ public newSounds: string[] = [];
+ public newImages: string[] = [];
+ private soundFiles = ['Scn2ALL.wav', 'Huh_sigh.wav', 'HMMMM.wav', 'Scn4bALL.wav', 'Sch1Whistle.wav', 'Scn4ALL.wav', '5.wav', 'Hmm.wav', '1.wav', 'Scn6Whistle.wav', 'Scn4Whistle.wav', 'Yay.wav', '3.wav', '4.wav', 'Scn6ALL.wav', '8.wav', 'mmMMmmm.wav', '9.wav', 'Scn2Whistle.wav', 'Scn6Voice.wav', '2.wav', 'NO.wav', '7.wav', 'emmemm.wav', 'Scn4bVoice.wav', 'Scn4Voice.wav', 'mumbleandhum.wav', 'Scn2Voice.wav', 'hello.wav', 'OK.wav', '6.wav', 'gotit.wav', 'Scn1ALL.wav', 'Scn1Voice.wav']
+ private imageFiles = [
+ "hannah.jpg",
+ "sad.png",
+ "13confused.png",
+ "newhannah.jpg",
+ "RGB24bits_320x240.png",
+ "1sleep.png",
+ "wink.png",
+ "panda.gif",
+ "sleep.png",
+ "angry.png",
+ "base_v2.png",
+ "surprise@2x.png",
+ "base@2x.png",
+ "annoyed.png",
+ "8confused.png",
+ "4asleep.png",
+ "13asleep.png",
+ "14asleep.png",
+ "colorcheck_320x240.png",
+ "3confused.png",
+ "4sleep.png",
+ "12asleep.png",
+ "2asleep.png",
+ "NTSCtest_320x240.png",
+ "happy.png",
+ "angry_RTeye_closed.bmp",
+ "15confused.png",
+ "asleep.png",
+ "angry_mouth.bmp",
+ "11asleep.png",
+ "6confused.png",
+ "9confused.png",
+ "disgust.png",
+ "angry_LTeye-closed.bmp",
+ "animesmileinvertedsmall.png",
+ "love.png",
+ "a.out",
+ "15asleep.png",
+ "14confused.png",
+ "db_animation-test.gif",
+ "5confused.png",
+ "PALtest_320x240.png",
+ "9asleep.png",
+ "surprise.png",
+ "5asleep.png",
+ "sadface.png",
+ "10asleep.png",
+ "3asleep.png",
+ "10confused.png",
+ "2confused.png",
+ "engaged.png",
+ "angry_mouth_closed.bmp",
+ "RGBParrot_320x240.png",
+ "page7orig.jpg",
+ "somethingWrong.png",
+ "confused.png",
+ "11confused.png",
+ "ball.gif",
+ "7confused.png",
+ "12confused.png",
+ "worried.png",
+ "animesmileinverted.png",
+ "angry_LTeye.bmp",
+ "7asleep.png",
+ "panda.jpg",
+ "animesmile.png",
+ "cambridge24bit_320x240.png",
+ "base_transparent@2x.png",
+ "8asleep.png",
+ "6asleep.png",
+ "fear.png",
+ "1asleep.png",
+ "3sleep.png",
+ "angry_cheek.bmp",
+ "4confused.png",
+ "base_v1.png",
+ "2sleep.png",
+ "1confused.png",
+ "angry_RTeye.bmp"
+ ];
private sensorData = ({
bumper: { front: 0, back: 0 },
altimeter: 0,
@@ -282,8 +366,14 @@ export default class Doodlebot {
private async onWebsocketMessage(event: MessageEvent) {
console.log("websocket message", { event });
- const text = await event.data.text();
- console.log(text);
+ try {
+ const text = await event.data.text();
+ console.log(text);
+ }
+ catch (e) {
+ console.log(JSON.stringify(event));
+ console.log("Error receiving message: ", e);
+ }
}
private invalidateWifiConnection() {
@@ -335,6 +425,25 @@ export default class Doodlebot {
return this.sensorData[type];
}
+ async findImageFiles() {
+ while (!this.connection) {
+ await new Promise(resolve => setTimeout(resolve, 100));
+ }
+ let endpoint = "http://" + this.connection.ip + ":8080/images/"
+ let uploadedImages = await this.fetchAndExtractList(endpoint);
+ return uploadedImages.filter(item => !this.imageFiles.includes(item));
+ }
+
+ async findSoundFiles() {
+ while (!this.connection) {
+ await new Promise(resolve => setTimeout(resolve, 100));
+ }
+ if (!this.connection) return [];
+ let endpoint = "http://" + this.connection.ip + ":8080/sounds/"
+ let uploadedSounds = await this.fetchAndExtractList(endpoint);
+ return uploadedSounds.filter(item => !this.soundFiles.includes(item));
+ }
+
/**
*
* @param type
@@ -440,7 +549,7 @@ export default class Doodlebot {
}
}
- setIP(ip: string) {
+ async setIP(ip: string) {
this.connection ??= { ip };
this.saveIP(ip);
return this.connection.ip = ip;
@@ -519,6 +628,97 @@ export default class Doodlebot {
});
}
+ getStoredIPAddress() {
+ if (!this.connection) { return "" }
+ return this.connection.ip;
+ }
+
+ parseWavHeader(uint8Array) {
+ const dataView = new DataView(uint8Array.buffer);
+
+ // Extract sample width, number of channels, and sample rate
+ const sampleWidth = dataView.getUint16(34, true) / 8; // Sample width in bytes (16-bit samples = 2 bytes, etc.)
+ const channels = dataView.getUint16(22, true); // Number of channels
+ const rate = dataView.getUint32(24, true); // Sample rate
+ const byteRate = dataView.getUint32(28, true); // Byte rate
+ const blockAlign = dataView.getUint16(32, true); // Block align
+ const dataSize = dataView.getUint32(40, true); // Size of the data chunk
+
+ const frameSize = blockAlign; // Size of each frame in bytes
+
+ return {
+ sampleWidth,
+ channels,
+ rate,
+ frameSize,
+ dataSize
+ };
+ }
+
+ splitIntoChunks(uint8Array, framesPerChunk) {
+ const headerInfo = this.parseWavHeader(uint8Array);
+ const { frameSize } = headerInfo;
+ const chunkSize = framesPerChunk * frameSize; // Number of bytes per chunk
+ const chunks = [];
+
+ // Skip the header (typically 44 bytes)
+ const dataStart = 44;
+
+ for (let i = dataStart; i < uint8Array.length; i += chunkSize) {
+ const chunk = uint8Array.slice(i, i + chunkSize);
+ chunks.push(chunk);
+ }
+
+ return chunks;
+ }
+
+
+ async sendAudioData(uint8Array: Uint8Array) {
+ let CHUNK_SIZE = 1024;
+ let ip = this.connection.ip;
+ const ws = makeWebsocket(ip, '8877');
+
+ ws.onopen = () => {
+ console.log('WebSocket connection opened');
+ let { sampleWidth, channels, rate } = this.parseWavHeader(uint8Array);
+ let first = "(1," + String(sampleWidth) + "," + String(channels) + "," + String(rate) + ")";
+ console.log(first);
+ ws.send(first);
+ let chunks = this.splitIntoChunks(uint8Array, CHUNK_SIZE);
+ let i = 0;
+ async function sendNextChunk() {
+ if (i >= chunks.length) {
+ console.log('All data sent');
+ ws.close();
+ return;
+ }
+
+ const chunk = chunks[i];
+
+ const binaryString = Array.from(chunk).map((byte: any) => String.fromCharCode(byte)).join('');;
+ const base64Data = btoa(binaryString);
+ const jsonData = JSON.stringify({ audio_data: base64Data });
+ ws.send(jsonData);
+ i = i + 1;
+ sendNextChunk();
+ }
+
+ sendNextChunk();
+ };
+
+ ws.onerror = (error) => {
+ console.error('WebSocket error:', error);
+ };
+
+ ws.onmessage = (message) => {
+ console.log(message);
+ }
+
+ ws.onclose = () => {
+ console.log('WebSocket connection closed');
+ };
+ }
+
/**
*
* @param credentials
@@ -627,10 +827,46 @@ export default class Doodlebot {
await this.sendWebsocketCommand(command.display, value);
}
- async displayText(text: string) {
+ async displayFile(file: string) {
+ await this.sendWebsocketCommand(command.display, file);
+ }
+
+ // Function to fetch and parse HTML template
+ async fetchAndExtractList(endpoint) {
+ try {
+ // Fetch the HTML template from the endpoint
+ const response = await fetch(endpoint);
+ // if (!response.ok) {
+ // throw new Error('Network response was not ok');
+ // }
+
+ // Get the HTML text
+ const htmlText = await response.text();
+
+ // Parse the HTML text into a DOM structure
+ const parser = new DOMParser();
+ const doc = parser.parseFromString(htmlText, 'text/html');
+
+ // Extract all elements
+ const listItems = doc.querySelectorAll('li');
+
+ // Get the text content of each element
+ const itemNames = Array.from(listItems).map(li => li.textContent.trim());
+
+ return itemNames;
+ } catch (error) {
+ throw new Error('Error fetching or parsing HTML:', error)
+ }
+ }
+
+ async displayText(text: string, size: string) {
+ //(d,F,[number]) == s, m, or l
+ await this.sendWebsocketCommand(command.display, "F", size);
await this.sendWebsocketCommand(command.display, "t", text);
}
+
+
/**
* NOTE: Consider making private
* @param command
diff --git a/extensions/src/doodlebot/FileArgument.svelte b/extensions/src/doodlebot/FileArgument.svelte
new file mode 100644
index 000000000..589002f9c
--- /dev/null
+++ b/extensions/src/doodlebot/FileArgument.svelte
@@ -0,0 +1,90 @@
+
+
+
+
+
+
+
+
diff --git a/extensions/src/doodlebot/ImageArgument.svelte b/extensions/src/doodlebot/ImageArgument.svelte
new file mode 100644
index 000000000..3785dc036
--- /dev/null
+++ b/extensions/src/doodlebot/ImageArgument.svelte
@@ -0,0 +1,73 @@
+
+
+
+
+
diff --git a/extensions/src/doodlebot/UI.svelte b/extensions/src/doodlebot/UI.svelte
new file mode 100644
index 000000000..16e9a9fea
--- /dev/null
+++ b/extensions/src/doodlebot/UI.svelte
@@ -0,0 +1,101 @@
+
+
+
+
Hello, world!
+ I am {extension.name}.
+
+
+
+
diff --git a/extensions/src/doodlebot/enums.ts b/extensions/src/doodlebot/enums.ts
index a7016bbc1..19542ea44 100644
--- a/extensions/src/doodlebot/enums.ts
+++ b/extensions/src/doodlebot/enums.ts
@@ -30,6 +30,7 @@ export const command = {
display: "d",
pen: "u",
network: "g",
+ speaker: "s"
} as const;
export type CommandKey = keyof typeof command;
diff --git a/extensions/src/doodlebot/index.ts b/extensions/src/doodlebot/index.ts
index f4ce9c186..480d1969a 100644
--- a/extensions/src/doodlebot/index.ts
+++ b/extensions/src/doodlebot/index.ts
@@ -1,7 +1,8 @@
-import { Environment, ExtensionMenuDisplayDetails, extension, block, buttonBlock } from "$common";
+import { Environment, ExtensionMenuDisplayDetails, extension, block, buttonBlock, scratch, BlockUtilityWithID } from "$common";
import { DisplayKey, displayKeys, command, type Command, SensorKey, sensorKeys } from "./enums";
import Doodlebot from "./Doodlebot";
import { splitArgsString } from "./utils";
+import FileArgument from './FileArgument.svelte';
import EventEmitter from "events";
import { categoryByGesture, classes, emojiByGesture, gestureDetection, gestureMenuItems, gestures, objectDetection } from "./detection";
@@ -35,7 +36,10 @@ const looper = (action: () => Promise, profileMarker?: string) => {
return controller;
}
-export default class DoodlebotBlocks extends extension(details, "ui", "indicators", "video", "drawable") {
+export var imageFiles = [];
+export var soundFiles: string[] = [];
+
+export default class DoodlebotBlocks extends extension(details, "ui", "customArguments", "indicators", "video", "drawable") {
doodlebot: Doodlebot;
private indicator: Promise<{ close(): void; }>;
@@ -56,18 +60,126 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator
imageStream: HTMLImageElement;
videoDrawable: ReturnType;
+ ip: string;
+ soundDictionary;
+ costumeDictionary: any;
+
- init(env: Environment) {
+ async init(env: Environment) {
this.openUI("Connect");
this.setIndicator("disconnected");
+ this.soundDictionary = {};
+ this.costumeDictionary = {};
+
+ env.runtime.on("TARGETS_UPDATE", async () => {
+ await this.setDictionaries();
+ })
+
+ await this.setDictionaries();
+
+ soundFiles = ["File"];
+ imageFiles = ["File"];
// idea: set up polling mechanism to try and disable unused sensors
// idea: set up polling mechanism to destroy gesture recognition loop
}
- setDoodlebot(doodlebot: Doodlebot) {
+ async setDictionaries() {
+ for (const target of this.runtime.targets) {
+ this.soundDictionary[target.id] = {};
+ this.costumeDictionary[target.id] = {};
+ if (target.sprite) {
+ for (const sound of target.sprite.sounds) {
+ if (sound.asset.dataFormat == "wav") {
+ this.soundDictionary[target.id][sound.name] = sound.asset.data;
+ }
+ }
+ for (const costume of target.sprite.costumes) {
+ let id = "Costume: " + costume.name;
+ if (costume.asset.dataFormat == "svg") {
+ await this.convertSvgUint8ArrayToPng(costume.asset.data, costume.size[0], costume.size[1])
+ .then((pngBlob: Blob) => {
+ const url = URL.createObjectURL(pngBlob)
+ this.costumeDictionary[target.id][id] = "costume9999.png---name---" + url;
+ })
+ } else if (costume.asset.dataFormat == "png") {
+ const blob = new Blob([costume.asset.data], { type: 'image/png' });
+ const url = URL.createObjectURL(blob)
+ this.costumeDictionary[target.id][id] = "costume9999.png---name---" + url;
+ }
+
+ }
+ }
+ }
+ }
+
+ async setIP(ip: string) {
+ this.ip = ip;
+
+ }
+
+ async convertSvgUint8ArrayToPng(uint8Array, width, height) {
+ return new Promise((resolve, reject) => {
+ // Convert Uint8Array to a string
+ const svgString = new TextDecoder().decode(uint8Array);
+
+ // Create an SVG Blob
+ const svgBlob = new Blob([svgString], { type: 'image/svg+xml;charset=utf-8' });
+ const url = URL.createObjectURL(svgBlob);
+
+ // Create an Image element
+ const img = new Image();
+ img.onload = () => {
+ // Create a canvas element
+ const canvas = document.createElement('canvas');
+ canvas.width = width;
+ canvas.height = height;
+ const ctx = canvas.getContext('2d');
+
+ // Draw the image on the canvas
+ ctx.drawImage(img, 0, 0, width, height);
+
+ // Convert the canvas to PNG data URL
+ const pngDataUrl = canvas.toDataURL('image/png');
+
+ // Convert the data URL to a Blob
+ fetch(pngDataUrl)
+ .then(res => res.blob())
+ .then(blob => {
+ // Clean up
+ URL.revokeObjectURL(url);
+ resolve(blob);
+ })
+ .catch(err => {
+ URL.revokeObjectURL(url);
+ reject(err);
+ });
+ };
+
+ img.onerror = reject;
+ img.src = url;
+ });
+ }
+
+ getCurrentSounds(id): string[] {
+ return Object.keys(this.soundDictionary[id]);
+ }
+
+ async setDoodlebot(doodlebot: Doodlebot) {
this.doodlebot = doodlebot;
this.setIndicator("connected");
+ try {
+ imageFiles = await doodlebot.findImageFiles();
+ soundFiles = await doodlebot.findSoundFiles();
+ } catch (e) {
+ this.openUI("ArrayError");
+ }
+ console.log("SETTING");
+ console.log(soundFiles);
+ }
+
+ async getIPAddress() {
+ return this.doodlebot?.getStoredIPAddress();
}
async setIndicator(status: "connected" | "disconnected") {
@@ -102,16 +214,17 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator
@block({
type: "command",
- text: (direction, steps) => `drive ${direction} for ${steps} steps`,
+ text: (direction, steps, speed) => `drive ${direction} for ${steps} steps at speed ${speed}`,
args: [
{ type: "string", options: ["forward", "backward", "left", "right"], defaultValue: "forward" },
- { type: "number", defaultValue: 2000 }
+ { type: "number", defaultValue: 2000 },
+ { type: "number", options: [1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000], defaultValue: 2000 }
]
})
- async drive(direction: "left" | "right" | "forward" | "backward", steps: number) {
+ async drive(direction: "left" | "right" | "forward" | "backward", steps: number, speed: number) {
const leftSteps = direction == "left" || direction == "backward" ? -steps : steps;
const rightSteps = direction == "right" || direction == "backward" ? -steps : steps;
- const stepsPerSecond = 2000;
+ const stepsPerSecond = speed;
await this.doodlebot?.motorCommand(
"steps",
@@ -226,22 +339,38 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator
await this.doodlebot?.disableSensor(sensor);
}
- @block({
+ @block((self) => ({
type: "command",
- text: (type: DisplayKey) => `display ${type}`,
- arg: { type: "string", options: displayKeys.filter(key => key !== "clear"), defaultValue: "happy" }
- })
- async setDisplay(display: DisplayKey) {
- await this.doodlebot?.display(display);
+ text: (type: DisplayKey | string) => `display ${type}`,
+ arg: {
+ type: "string", options: () => {
+ self.setDictionaries();
+ return displayKeys.filter(key => key !== "clear").concat(imageFiles).concat(Object.keys(self.costumeDictionary[self.runtime._editingTarget.id]) as any[]).filter((item: string) => item != "costume9999.png")
+ }, defaultValue: "happy"
+ }
+ }))
+ async setDisplay(display: DisplayKey | string) {
+ let costumeNames = Object.keys(this.costumeDictionary[this.runtime._editingTarget.id]);
+ if (costumeNames.includes(display)) {
+ await this.uploadFile("image", this.costumeDictionary[this.runtime._editingTarget.id][display]);
+ await this.setArrays();
+ await this.doodlebot.displayFile("costume9999.png");
+ } else if (imageFiles.includes(display)) {
+ await this.doodlebot?.displayFile(display);
+ } else {
+ await this.doodlebot?.display(display as DisplayKey);
+ }
+
}
@block({
type: "command",
- text: (text: string) => `display text ${text}`,
- arg: { type: "string", defaultValue: "hello world!" }
+ text: (text: string, size: string) => `display text ${text} with size ${size}`,
+ args: [{ type: "string", defaultValue: "hello world!" },
+ { type: "string", options: ["s", "m", "l"], defaultValue: "m" }]
})
- async setText(text: string) {
- await this.doodlebot?.displayText(text);
+ async setText(text: string, size: string) {
+ await this.doodlebot?.displayText(text, size);
}
@block({
@@ -261,6 +390,27 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator
await this.doodlebot?.sendWebsocketCommand("m", sound)
}
+ @block((self) => ({
+ type: "command",
+ text: (sound) => `play sound ${sound}`,
+ arg: {
+ type: "string", options: () => soundFiles.concat(self.getCurrentSounds(self.runtime._editingTarget.id))
+ }
+ }))
+ async playSoundFile(sound: string, util: BlockUtilityWithID) {
+ let currentId = this.runtime._editingTarget.id;
+ let costumeSounds = this.getCurrentSounds(currentId);
+ if (costumeSounds.includes(sound)) {
+ let soundArray = this.soundDictionary[currentId][sound];
+ console.log(soundArray);
+ await this.doodlebot.sendAudioData(soundArray);
+ } else {
+ await this.doodlebot?.sendWebsocketCommand("m", sound)
+ }
+
+ }
+
+
@block({
type: "command",
text: (transparency) => `display video with ${transparency}% transparency`,
@@ -344,6 +494,12 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator
await new Promise((resolve) => setTimeout(resolve, audioDuration * 1000));
}
+ async setArrays() {
+ imageFiles = await this.doodlebot.findImageFiles();
+ soundFiles = await this.doodlebot.findSoundFiles();
+ console.log("SETTING");
+ }
+
@block({
type: "reporter",
@@ -353,6 +509,68 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator
return this.doodlebot?.getIPAddress();
}
+ // @block({
+ // type: "command",
+ // text: "Upload files"
+ // })
+ // async uploadFiles() {
+ // this.openUI("UI");
+ // }
+
+ async uploadFile(type: string, blobURL: string) {
+ const ip = await this.getIPAddress();
+ let uploadEndpoint;
+ if (type == "sound") {
+ uploadEndpoint = "http://" + ip + ":8080/sounds_upload";
+ } else {
+ uploadEndpoint = "http://" + ip + ":8080/img_upload";
+ }
+
+ try {
+ const components = blobURL.split("---name---");
+ console.log("COMPONENTS");
+ console.log(components);
+ const response1 = await fetch(components[1]);
+ if (!response1.ok) {
+ throw new Error(`Failed to fetch Blob from URL: ${blobURL}`);
+ }
+ const blob = await response1.blob();
+ // Convert Blob to File
+ const file = new File([blob], components[0], { type: blob.type });
+ const formData = new FormData();
+ formData.append("file", file);
+
+ console.log("file");
+ console.log(file);
+
+ const response2 = await fetch(uploadEndpoint, {
+ method: "POST",
+ body: formData,
+ });
+
+ console.log(response2);
+
+ if (!response2.ok) {
+ throw new Error(`Failed to upload file: ${response2.statusText}`);
+ }
+
+ console.log("File uploaded successfully");
+ this.setArrays();
+ } catch (error) {
+ console.error("Error:", error);
+ }
+ }
+
+ @(scratch.command((self, $) => $`Upload sound file ${self.makeCustomArgument({ component: FileArgument, initial: { value: "", text: "File" } })}`))
+ async uploadSoundFile(test: string) {
+ await this.uploadFile("sound", test);
+ }
+
+ @(scratch.command((self, $) => $`Upload image file ${self.makeCustomArgument({ component: FileArgument, initial: { value: "", text: "File" } })}`))
+ async uploadImageFile(test: string) {
+ await this.uploadFile("image", test);
+ }
+
@block({
type: "command",
text: (_command, args, protocol) => `send (${_command}, ${args}) over ${protocol}`,