diff --git a/extensions/src/doodlebot/ArrayError.svelte b/extensions/src/doodlebot/ArrayError.svelte new file mode 100644 index 000000000..497616f7c --- /dev/null +++ b/extensions/src/doodlebot/ArrayError.svelte @@ -0,0 +1,40 @@ + + +
+

Cannot load Doodlebot's sound/image server!

+

Please try reloading the page.

+
+ + diff --git a/extensions/src/doodlebot/CustomArgument.svelte b/extensions/src/doodlebot/CustomArgument.svelte new file mode 100644 index 000000000..f2dbd9189 --- /dev/null +++ b/extensions/src/doodlebot/CustomArgument.svelte @@ -0,0 +1,83 @@ + + +
+ {#each soundFiles as f} + {#if value == f} +
+
+
+ +
+
+ {/if} + {#if value != f} +
+
+ +
+
+ {/if} + {/each} +
+ + diff --git a/extensions/src/doodlebot/Doodlebot.ts b/extensions/src/doodlebot/Doodlebot.ts index 33de6ed72..111f207c8 100644 --- a/extensions/src/doodlebot/Doodlebot.ts +++ b/extensions/src/doodlebot/Doodlebot.ts @@ -147,6 +147,90 @@ export default class Doodlebot { private isStopped = true; // should this be initializeed more intelligently? + public newSounds: string[] = []; + public newImages: string[] = []; + private soundFiles = ['Scn2ALL.wav', 'Huh_sigh.wav', 'HMMMM.wav', 'Scn4bALL.wav', 'Sch1Whistle.wav', 'Scn4ALL.wav', '5.wav', 'Hmm.wav', '1.wav', 'Scn6Whistle.wav', 'Scn4Whistle.wav', 'Yay.wav', '3.wav', '4.wav', 'Scn6ALL.wav', '8.wav', 'mmMMmmm.wav', '9.wav', 'Scn2Whistle.wav', 'Scn6Voice.wav', '2.wav', 'NO.wav', '7.wav', 'emmemm.wav', 'Scn4bVoice.wav', 'Scn4Voice.wav', 'mumbleandhum.wav', 'Scn2Voice.wav', 'hello.wav', 'OK.wav', '6.wav', 'gotit.wav', 'Scn1ALL.wav', 'Scn1Voice.wav'] + private imageFiles = [ + "hannah.jpg", + "sad.png", + "13confused.png", + "newhannah.jpg", + "RGB24bits_320x240.png", + "1sleep.png", + "wink.png", + "panda.gif", + "sleep.png", + "angry.png", + "base_v2.png", + "surprise@2x.png", + "base@2x.png", + "annoyed.png", + "8confused.png", + "4asleep.png", + "13asleep.png", + "14asleep.png", + "colorcheck_320x240.png", + "3confused.png", + "4sleep.png", + "12asleep.png", + "2asleep.png", + "NTSCtest_320x240.png", + "happy.png", + "angry_RTeye_closed.bmp", + "15confused.png", + "asleep.png", + "angry_mouth.bmp", + "11asleep.png", + "6confused.png", + "9confused.png", + "disgust.png", + "angry_LTeye-closed.bmp", + "animesmileinvertedsmall.png", + "love.png", + "a.out", + "15asleep.png", + "14confused.png", + "db_animation-test.gif", + "5confused.png", + "PALtest_320x240.png", + "9asleep.png", + "surprise.png", + "5asleep.png", + "sadface.png", + "10asleep.png", + "3asleep.png", + "10confused.png", + "2confused.png", + "engaged.png", + "angry_mouth_closed.bmp", + "RGBParrot_320x240.png", + "page7orig.jpg", + "somethingWrong.png", + "confused.png", + "11confused.png", + "ball.gif", + "7confused.png", + "12confused.png", + "worried.png", + "animesmileinverted.png", + "angry_LTeye.bmp", + "7asleep.png", + "panda.jpg", + "animesmile.png", + "cambridge24bit_320x240.png", + "base_transparent@2x.png", + "8asleep.png", + "6asleep.png", + "fear.png", + "1asleep.png", + "3sleep.png", + "angry_cheek.bmp", + "4confused.png", + "base_v1.png", + "2sleep.png", + "1confused.png", + "angry_RTeye.bmp" + ]; private sensorData = ({ bumper: { front: 0, back: 0 }, altimeter: 0, @@ -282,8 +366,14 @@ export default class Doodlebot { private async onWebsocketMessage(event: MessageEvent) { console.log("websocket message", { event }); - const text = await event.data.text(); - console.log(text); + try { + const text = await event.data.text(); + console.log(text); + } + catch (e) { + console.log(JSON.stringify(event)); + console.log("Error receiving message: ", e); + } } private invalidateWifiConnection() { @@ -335,6 +425,25 @@ export default class Doodlebot { return this.sensorData[type]; } + async findImageFiles() { + while (!this.connection) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + let endpoint = "http://" + this.connection.ip + ":8080/images/" + let uploadedImages = await this.fetchAndExtractList(endpoint); + return uploadedImages.filter(item => !this.imageFiles.includes(item)); + } + + async findSoundFiles() { + while (!this.connection) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + if (!this.connection) return []; + let endpoint = "http://" + this.connection.ip + ":8080/sounds/" + let uploadedSounds = await this.fetchAndExtractList(endpoint); + return uploadedSounds.filter(item => !this.soundFiles.includes(item)); + } + /** * * @param type @@ -440,7 +549,7 @@ export default class Doodlebot { } } - setIP(ip: string) { + async setIP(ip: string) { this.connection ??= { ip }; this.saveIP(ip); return this.connection.ip = ip; @@ -519,6 +628,97 @@ export default class Doodlebot { }); } + getStoredIPAddress() { + if (!this.connection) { return "" } + return this.connection.ip; + } + + parseWavHeader(uint8Array) { + const dataView = new DataView(uint8Array.buffer); + + // Extract sample width, number of channels, and sample rate + const sampleWidth = dataView.getUint16(34, true) / 8; // Sample width in bytes (16-bit samples = 2 bytes, etc.) + const channels = dataView.getUint16(22, true); // Number of channels + const rate = dataView.getUint32(24, true); // Sample rate + const byteRate = dataView.getUint32(28, true); // Byte rate + const blockAlign = dataView.getUint16(32, true); // Block align + const dataSize = dataView.getUint32(40, true); // Size of the data chunk + + const frameSize = blockAlign; // Size of each frame in bytes + + return { + sampleWidth, + channels, + rate, + frameSize, + dataSize + }; + } + + splitIntoChunks(uint8Array, framesPerChunk) { + const headerInfo = this.parseWavHeader(uint8Array); + const { frameSize } = headerInfo; + const chunkSize = framesPerChunk * frameSize; // Number of bytes per chunk + const chunks = []; + + // Skip the header (typically 44 bytes) + const dataStart = 44; + + for (let i = dataStart; i < uint8Array.length; i += chunkSize) { + const chunk = uint8Array.slice(i, i + chunkSize); + chunks.push(chunk); + } + + return chunks; + } + + + async sendAudioData(uint8Array: Uint8Array) { + let CHUNK_SIZE = 1024; + let ip = this.connection.ip; + const ws = makeWebsocket(ip, '8877'); + + ws.onopen = () => { + console.log('WebSocket connection opened'); + let { sampleWidth, channels, rate } = this.parseWavHeader(uint8Array); + let first = "(1," + String(sampleWidth) + "," + String(channels) + "," + String(rate) + ")"; + console.log(first); + ws.send(first); + let chunks = this.splitIntoChunks(uint8Array, CHUNK_SIZE); + let i = 0; + async function sendNextChunk() { + if (i >= chunks.length) { + console.log('All data sent'); + ws.close(); + return; + } + + const chunk = chunks[i]; + + const binaryString = Array.from(chunk).map((byte: any) => String.fromCharCode(byte)).join('');; + const base64Data = btoa(binaryString); + const jsonData = JSON.stringify({ audio_data: base64Data }); + ws.send(jsonData); + i = i + 1; + sendNextChunk(); + } + + sendNextChunk(); + }; + + ws.onerror = (error) => { + console.error('WebSocket error:', error); + }; + + ws.onmessage = (message) => { + console.log(message); + } + + ws.onclose = () => { + console.log('WebSocket connection closed'); + }; + } + /** * * @param credentials @@ -627,10 +827,46 @@ export default class Doodlebot { await this.sendWebsocketCommand(command.display, value); } - async displayText(text: string) { + async displayFile(file: string) { + await this.sendWebsocketCommand(command.display, file); + } + + // Function to fetch and parse HTML template + async fetchAndExtractList(endpoint) { + try { + // Fetch the HTML template from the endpoint + const response = await fetch(endpoint); + // if (!response.ok) { + // throw new Error('Network response was not ok'); + // } + + // Get the HTML text + const htmlText = await response.text(); + + // Parse the HTML text into a DOM structure + const parser = new DOMParser(); + const doc = parser.parseFromString(htmlText, 'text/html'); + + // Extract all
  • elements + const listItems = doc.querySelectorAll('li'); + + // Get the text content of each
  • element + const itemNames = Array.from(listItems).map(li => li.textContent.trim()); + + return itemNames; + } catch (error) { + throw new Error('Error fetching or parsing HTML:', error) + } + } + + async displayText(text: string, size: string) { + //(d,F,[number]) == s, m, or l + await this.sendWebsocketCommand(command.display, "F", size); await this.sendWebsocketCommand(command.display, "t", text); } + + /** * NOTE: Consider making private * @param command diff --git a/extensions/src/doodlebot/FileArgument.svelte b/extensions/src/doodlebot/FileArgument.svelte new file mode 100644 index 000000000..589002f9c --- /dev/null +++ b/extensions/src/doodlebot/FileArgument.svelte @@ -0,0 +1,90 @@ + + +
    + + +
    + + diff --git a/extensions/src/doodlebot/ImageArgument.svelte b/extensions/src/doodlebot/ImageArgument.svelte new file mode 100644 index 000000000..3785dc036 --- /dev/null +++ b/extensions/src/doodlebot/ImageArgument.svelte @@ -0,0 +1,73 @@ + + + + + diff --git a/extensions/src/doodlebot/UI.svelte b/extensions/src/doodlebot/UI.svelte new file mode 100644 index 000000000..16e9a9fea --- /dev/null +++ b/extensions/src/doodlebot/UI.svelte @@ -0,0 +1,101 @@ + + +
    +

    Hello, world!

    +

    I am {extension.name}.

    + +
    + + diff --git a/extensions/src/doodlebot/enums.ts b/extensions/src/doodlebot/enums.ts index a7016bbc1..19542ea44 100644 --- a/extensions/src/doodlebot/enums.ts +++ b/extensions/src/doodlebot/enums.ts @@ -30,6 +30,7 @@ export const command = { display: "d", pen: "u", network: "g", + speaker: "s" } as const; export type CommandKey = keyof typeof command; diff --git a/extensions/src/doodlebot/index.ts b/extensions/src/doodlebot/index.ts index f4ce9c186..480d1969a 100644 --- a/extensions/src/doodlebot/index.ts +++ b/extensions/src/doodlebot/index.ts @@ -1,7 +1,8 @@ -import { Environment, ExtensionMenuDisplayDetails, extension, block, buttonBlock } from "$common"; +import { Environment, ExtensionMenuDisplayDetails, extension, block, buttonBlock, scratch, BlockUtilityWithID } from "$common"; import { DisplayKey, displayKeys, command, type Command, SensorKey, sensorKeys } from "./enums"; import Doodlebot from "./Doodlebot"; import { splitArgsString } from "./utils"; +import FileArgument from './FileArgument.svelte'; import EventEmitter from "events"; import { categoryByGesture, classes, emojiByGesture, gestureDetection, gestureMenuItems, gestures, objectDetection } from "./detection"; @@ -35,7 +36,10 @@ const looper = (action: () => Promise, profileMarker?: string) => { return controller; } -export default class DoodlebotBlocks extends extension(details, "ui", "indicators", "video", "drawable") { +export var imageFiles = []; +export var soundFiles: string[] = []; + +export default class DoodlebotBlocks extends extension(details, "ui", "customArguments", "indicators", "video", "drawable") { doodlebot: Doodlebot; private indicator: Promise<{ close(): void; }>; @@ -56,18 +60,126 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator imageStream: HTMLImageElement; videoDrawable: ReturnType; + ip: string; + soundDictionary; + costumeDictionary: any; + - init(env: Environment) { + async init(env: Environment) { this.openUI("Connect"); this.setIndicator("disconnected"); + this.soundDictionary = {}; + this.costumeDictionary = {}; + + env.runtime.on("TARGETS_UPDATE", async () => { + await this.setDictionaries(); + }) + + await this.setDictionaries(); + + soundFiles = ["File"]; + imageFiles = ["File"]; // idea: set up polling mechanism to try and disable unused sensors // idea: set up polling mechanism to destroy gesture recognition loop } - setDoodlebot(doodlebot: Doodlebot) { + async setDictionaries() { + for (const target of this.runtime.targets) { + this.soundDictionary[target.id] = {}; + this.costumeDictionary[target.id] = {}; + if (target.sprite) { + for (const sound of target.sprite.sounds) { + if (sound.asset.dataFormat == "wav") { + this.soundDictionary[target.id][sound.name] = sound.asset.data; + } + } + for (const costume of target.sprite.costumes) { + let id = "Costume: " + costume.name; + if (costume.asset.dataFormat == "svg") { + await this.convertSvgUint8ArrayToPng(costume.asset.data, costume.size[0], costume.size[1]) + .then((pngBlob: Blob) => { + const url = URL.createObjectURL(pngBlob) + this.costumeDictionary[target.id][id] = "costume9999.png---name---" + url; + }) + } else if (costume.asset.dataFormat == "png") { + const blob = new Blob([costume.asset.data], { type: 'image/png' }); + const url = URL.createObjectURL(blob) + this.costumeDictionary[target.id][id] = "costume9999.png---name---" + url; + } + + } + } + } + } + + async setIP(ip: string) { + this.ip = ip; + + } + + async convertSvgUint8ArrayToPng(uint8Array, width, height) { + return new Promise((resolve, reject) => { + // Convert Uint8Array to a string + const svgString = new TextDecoder().decode(uint8Array); + + // Create an SVG Blob + const svgBlob = new Blob([svgString], { type: 'image/svg+xml;charset=utf-8' }); + const url = URL.createObjectURL(svgBlob); + + // Create an Image element + const img = new Image(); + img.onload = () => { + // Create a canvas element + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext('2d'); + + // Draw the image on the canvas + ctx.drawImage(img, 0, 0, width, height); + + // Convert the canvas to PNG data URL + const pngDataUrl = canvas.toDataURL('image/png'); + + // Convert the data URL to a Blob + fetch(pngDataUrl) + .then(res => res.blob()) + .then(blob => { + // Clean up + URL.revokeObjectURL(url); + resolve(blob); + }) + .catch(err => { + URL.revokeObjectURL(url); + reject(err); + }); + }; + + img.onerror = reject; + img.src = url; + }); + } + + getCurrentSounds(id): string[] { + return Object.keys(this.soundDictionary[id]); + } + + async setDoodlebot(doodlebot: Doodlebot) { this.doodlebot = doodlebot; this.setIndicator("connected"); + try { + imageFiles = await doodlebot.findImageFiles(); + soundFiles = await doodlebot.findSoundFiles(); + } catch (e) { + this.openUI("ArrayError"); + } + console.log("SETTING"); + console.log(soundFiles); + } + + async getIPAddress() { + return this.doodlebot?.getStoredIPAddress(); } async setIndicator(status: "connected" | "disconnected") { @@ -102,16 +214,17 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator @block({ type: "command", - text: (direction, steps) => `drive ${direction} for ${steps} steps`, + text: (direction, steps, speed) => `drive ${direction} for ${steps} steps at speed ${speed}`, args: [ { type: "string", options: ["forward", "backward", "left", "right"], defaultValue: "forward" }, - { type: "number", defaultValue: 2000 } + { type: "number", defaultValue: 2000 }, + { type: "number", options: [1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000], defaultValue: 2000 } ] }) - async drive(direction: "left" | "right" | "forward" | "backward", steps: number) { + async drive(direction: "left" | "right" | "forward" | "backward", steps: number, speed: number) { const leftSteps = direction == "left" || direction == "backward" ? -steps : steps; const rightSteps = direction == "right" || direction == "backward" ? -steps : steps; - const stepsPerSecond = 2000; + const stepsPerSecond = speed; await this.doodlebot?.motorCommand( "steps", @@ -226,22 +339,38 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator await this.doodlebot?.disableSensor(sensor); } - @block({ + @block((self) => ({ type: "command", - text: (type: DisplayKey) => `display ${type}`, - arg: { type: "string", options: displayKeys.filter(key => key !== "clear"), defaultValue: "happy" } - }) - async setDisplay(display: DisplayKey) { - await this.doodlebot?.display(display); + text: (type: DisplayKey | string) => `display ${type}`, + arg: { + type: "string", options: () => { + self.setDictionaries(); + return displayKeys.filter(key => key !== "clear").concat(imageFiles).concat(Object.keys(self.costumeDictionary[self.runtime._editingTarget.id]) as any[]).filter((item: string) => item != "costume9999.png") + }, defaultValue: "happy" + } + })) + async setDisplay(display: DisplayKey | string) { + let costumeNames = Object.keys(this.costumeDictionary[this.runtime._editingTarget.id]); + if (costumeNames.includes(display)) { + await this.uploadFile("image", this.costumeDictionary[this.runtime._editingTarget.id][display]); + await this.setArrays(); + await this.doodlebot.displayFile("costume9999.png"); + } else if (imageFiles.includes(display)) { + await this.doodlebot?.displayFile(display); + } else { + await this.doodlebot?.display(display as DisplayKey); + } + } @block({ type: "command", - text: (text: string) => `display text ${text}`, - arg: { type: "string", defaultValue: "hello world!" } + text: (text: string, size: string) => `display text ${text} with size ${size}`, + args: [{ type: "string", defaultValue: "hello world!" }, + { type: "string", options: ["s", "m", "l"], defaultValue: "m" }] }) - async setText(text: string) { - await this.doodlebot?.displayText(text); + async setText(text: string, size: string) { + await this.doodlebot?.displayText(text, size); } @block({ @@ -261,6 +390,27 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator await this.doodlebot?.sendWebsocketCommand("m", sound) } + @block((self) => ({ + type: "command", + text: (sound) => `play sound ${sound}`, + arg: { + type: "string", options: () => soundFiles.concat(self.getCurrentSounds(self.runtime._editingTarget.id)) + } + })) + async playSoundFile(sound: string, util: BlockUtilityWithID) { + let currentId = this.runtime._editingTarget.id; + let costumeSounds = this.getCurrentSounds(currentId); + if (costumeSounds.includes(sound)) { + let soundArray = this.soundDictionary[currentId][sound]; + console.log(soundArray); + await this.doodlebot.sendAudioData(soundArray); + } else { + await this.doodlebot?.sendWebsocketCommand("m", sound) + } + + } + + @block({ type: "command", text: (transparency) => `display video with ${transparency}% transparency`, @@ -344,6 +494,12 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator await new Promise((resolve) => setTimeout(resolve, audioDuration * 1000)); } + async setArrays() { + imageFiles = await this.doodlebot.findImageFiles(); + soundFiles = await this.doodlebot.findSoundFiles(); + console.log("SETTING"); + } + @block({ type: "reporter", @@ -353,6 +509,68 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator return this.doodlebot?.getIPAddress(); } + // @block({ + // type: "command", + // text: "Upload files" + // }) + // async uploadFiles() { + // this.openUI("UI"); + // } + + async uploadFile(type: string, blobURL: string) { + const ip = await this.getIPAddress(); + let uploadEndpoint; + if (type == "sound") { + uploadEndpoint = "http://" + ip + ":8080/sounds_upload"; + } else { + uploadEndpoint = "http://" + ip + ":8080/img_upload"; + } + + try { + const components = blobURL.split("---name---"); + console.log("COMPONENTS"); + console.log(components); + const response1 = await fetch(components[1]); + if (!response1.ok) { + throw new Error(`Failed to fetch Blob from URL: ${blobURL}`); + } + const blob = await response1.blob(); + // Convert Blob to File + const file = new File([blob], components[0], { type: blob.type }); + const formData = new FormData(); + formData.append("file", file); + + console.log("file"); + console.log(file); + + const response2 = await fetch(uploadEndpoint, { + method: "POST", + body: formData, + }); + + console.log(response2); + + if (!response2.ok) { + throw new Error(`Failed to upload file: ${response2.statusText}`); + } + + console.log("File uploaded successfully"); + this.setArrays(); + } catch (error) { + console.error("Error:", error); + } + } + + @(scratch.command((self, $) => $`Upload sound file ${self.makeCustomArgument({ component: FileArgument, initial: { value: "", text: "File" } })}`)) + async uploadSoundFile(test: string) { + await this.uploadFile("sound", test); + } + + @(scratch.command((self, $) => $`Upload image file ${self.makeCustomArgument({ component: FileArgument, initial: { value: "", text: "File" } })}`)) + async uploadImageFile(test: string) { + await this.uploadFile("image", test); + } + @block({ type: "command", text: (_command, args, protocol) => `send (${_command}, ${args}) over ${protocol}`,