From 53425a64464428de2f219d94d64199b1e48f76a9 Mon Sep 17 00:00:00 2001 From: ymayarajan3 Date: Thu, 1 Aug 2024 15:17:58 -0400 Subject: [PATCH 01/11] mezzanine edits --- .../src/doodlebot/CustomArgument.svelte | 83 ++++++ extensions/src/doodlebot/Doodlebot.ts | 254 +++++++++++++++++- extensions/src/doodlebot/UI.svelte | 101 +++++++ extensions/src/doodlebot/enums.ts | 1 + extensions/src/doodlebot/index.ts | 127 ++++++++- 5 files changed, 551 insertions(+), 15 deletions(-) create mode 100644 extensions/src/doodlebot/CustomArgument.svelte create mode 100644 extensions/src/doodlebot/UI.svelte diff --git a/extensions/src/doodlebot/CustomArgument.svelte b/extensions/src/doodlebot/CustomArgument.svelte new file mode 100644 index 000000000..f2dbd9189 --- /dev/null +++ b/extensions/src/doodlebot/CustomArgument.svelte @@ -0,0 +1,83 @@ + + +
+ {#each soundFiles as f} + {#if value == f} +
+
+
+ +
+
+ {/if} + {#if value != f} +
+
+ +
+
+ {/if} + {/each} +
+ + diff --git a/extensions/src/doodlebot/Doodlebot.ts b/extensions/src/doodlebot/Doodlebot.ts index 33de6ed72..756c148df 100644 --- a/extensions/src/doodlebot/Doodlebot.ts +++ b/extensions/src/doodlebot/Doodlebot.ts @@ -147,6 +147,90 @@ export default class Doodlebot { private isStopped = true; // should this be initializeed more intelligently? + public newSounds: string[] = []; + public newImages: string[] = []; + private soundFiles = ['Scn2ALL.wav', 'Huh_sigh.wav', 'HMMMM.wav', 'Scn4bALL.wav', 'Sch1Whistle.wav', 'Scn4ALL.wav', '5.wav', 'Hmm.wav', '1.wav', 'Scn6Whistle.wav', 'Scn4Whistle.wav', 'Yay.wav', '3.wav', '4.wav', 'Scn6ALL.wav', '8.wav', 'mmMMmmm.wav', '9.wav', 'Scn2Whistle.wav', 'Scn6Voice.wav', '2.wav', 'NO.wav', '7.wav', 'emmemm.wav', 'Scn4bVoice.wav', 'Scn4Voice.wav', 'mumbleandhum.wav', 'Scn2Voice.wav', 'hello.wav', 'OK.wav', '6.wav', 'gotit.wav', 'Scn1ALL.wav', 'Scn1Voice.wav'] + private imageFiles = [ + "hannah.jpg", + "sad.png", + "13confused.png", + "newhannah.jpg", + "RGB24bits_320x240.png", + "1sleep.png", + "wink.png", + "panda.gif", + "sleep.png", + "angry.png", + "base_v2.png", + "surprise@2x.png", + "base@2x.png", + "annoyed.png", + "8confused.png", + "4asleep.png", + "13asleep.png", + "14asleep.png", + "colorcheck_320x240.png", + "3confused.png", + "4sleep.png", + "12asleep.png", + "2asleep.png", + "NTSCtest_320x240.png", + "happy.png", + "angry_RTeye_closed.bmp", + "15confused.png", + "asleep.png", + "angry_mouth.bmp", + "11asleep.png", + "6confused.png", + "9confused.png", + "disgust.png", + "angry_LTeye-closed.bmp", + "animesmileinvertedsmall.png", + "love.png", + "a.out", + "15asleep.png", + "14confused.png", + "db_animation-test.gif", + "5confused.png", + "PALtest_320x240.png", + "9asleep.png", + "surprise.png", + "5asleep.png", + "sadface.png", + "10asleep.png", + "3asleep.png", + "10confused.png", + "2confused.png", + "engaged.png", + "angry_mouth_closed.bmp", + "RGBParrot_320x240.png", + "page7orig.jpg", + "somethingWrong.png", + "confused.png", + "11confused.png", + "ball.gif", + "7confused.png", + "12confused.png", + "worried.png", + "animesmileinverted.png", + "angry_LTeye.bmp", + "7asleep.png", + "panda.jpg", + "animesmile.png", + "cambridge24bit_320x240.png", + "base_transparent@2x.png", + "8asleep.png", + "6asleep.png", + "fear.png", + "1asleep.png", + "3sleep.png", + "angry_cheek.bmp", + "4confused.png", + "base_v1.png", + "2sleep.png", + "1confused.png", + "angry_RTeye.bmp" + ]; private sensorData = ({ bumper: { front: 0, back: 0 }, altimeter: 0, @@ -282,8 +366,14 @@ export default class Doodlebot { private async onWebsocketMessage(event: MessageEvent) { console.log("websocket message", { event }); - const text = await event.data.text(); - console.log(text); + try { + const text = await event.data.text(); + console.log(text); + } + catch (e) { + console.log(JSON.stringify(event)); + console.log("Error receiving message: ", e); + } } private invalidateWifiConnection() { @@ -335,6 +425,29 @@ export default class Doodlebot { return this.sensorData[type]; } + async findImageFiles() { + while (!this.connection) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + let endpoint = "http://" + this.connection.ip + ":8080/images/" + console.log(endpoint); + let uploadedImages = await this.fetchAndExtractList(endpoint); + return uploadedImages.filter(item => !this.imageFiles.includes(item)); + } + + async findSoundFiles() { + while (!this.connection) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + if (!this.connection) return []; + let endpoint = "http://" + this.connection.ip + ":8080/sounds/" + let uploadedSounds = await this.fetchAndExtractList(endpoint); + console.log("uploaded"); + console.log(uploadedSounds); + console.log(uploadedSounds.filter(item => !this.soundFiles.includes(item))); + return uploadedSounds.filter(item => !this.soundFiles.includes(item)); + } + /** * * @param type @@ -440,7 +553,7 @@ export default class Doodlebot { } } - setIP(ip: string) { + async setIP(ip: string) { this.connection ??= { ip }; this.saveIP(ip); return this.connection.ip = ip; @@ -519,6 +632,103 @@ export default class Doodlebot { }); } + parseWavHeader(uint8Array) { + const dataView = new DataView(uint8Array.buffer); + + // Extract sample width, number of channels, and sample rate + const sampleWidth = dataView.getUint16(34, true) / 8; // Sample width in bytes (16-bit samples = 2 bytes, etc.) + const channels = dataView.getUint16(22, true); // Number of channels + const rate = dataView.getUint32(24, true); // Sample rate + const byteRate = dataView.getUint32(28, true); // Byte rate + const blockAlign = dataView.getUint16(32, true); // Block align + const dataSize = dataView.getUint32(40, true); // Size of the data chunk + + const frameSize = blockAlign; // Size of each frame in bytes + + return { + sampleWidth, + channels, + rate, + frameSize, + dataSize + }; + } + + splitIntoChunks(uint8Array, framesPerChunk) { + const headerInfo = this.parseWavHeader(uint8Array); + const { frameSize } = headerInfo; + const chunkSize = framesPerChunk * frameSize; // Number of bytes per chunk + const chunks = []; + + // Skip the header (typically 44 bytes) + const dataStart = 44; + + for (let i = dataStart; i < uint8Array.length; i += chunkSize) { + const chunk = uint8Array.slice(i, i + chunkSize); + chunks.push(chunk); + } + + return chunks; + } + + + async sendAudioData(uint8Array: Uint8Array) { + let CHUNK_SIZE = 1024; + let ip = this.connection.ip; + const ws = makeWebsocket(ip, '8877'); + + ws.onopen = () => { + console.log('WebSocket connection opened'); + + let offset = 0; + + let { sampleWidth, channels, rate } = this.parseWavHeader(uint8Array); + let first = "(1," + String(sampleWidth) + "," + String(channels) + "," + String(rate) + ")"; + console.log(first); + ws.send(first); + let chunks = this.splitIntoChunks(uint8Array, CHUNK_SIZE); + let i = 0; + async function sendNextChunk() { + if (i >= chunks.length) { + console.log('All data sent'); + ws.close(); + return; + } + + // Calculate end position of the current chunk + //const end = Math.min(offset + CHUNK_SIZE, uint8Array.length); + const chunk = chunks[i]; + console.log("sending"); + + const binaryString = Array.from(chunk).map((byte: any) => String.fromCharCode(byte)).join('');; + const base64Data = btoa(binaryString); + const jsonData = JSON.stringify({ audio_data: base64Data }); + console.log(jsonData); + ws.send(jsonData); + i = i + 1; + sendNextChunk(); + } + + // Start sending chunks + sendNextChunk(); + }; + + ws.onerror = (error) => { + console.error('WebSocket error:', error); + }; + + ws.onmessage = (event) => { + const response = JSON.parse(event.data); + if (response.type === 'ack' && response.status === 'received') { + console.log('Message successfully received by server'); + } + } + + ws.onclose = () => { + console.log('WebSocket connection closed'); + }; + } + /** * * @param credentials @@ -627,10 +837,46 @@ export default class Doodlebot { await this.sendWebsocketCommand(command.display, value); } - async displayText(text: string) { + // Function to fetch and parse HTML template + async fetchAndExtractList(endpoint) { + try { + // Fetch the HTML template from the endpoint + const response = await fetch(endpoint); + // if (!response.ok) { + // throw new Error('Network response was not ok'); + // } + + // Get the HTML text + const htmlText = await response.text(); + + console.log(response); + console.log(htmlText); + + // Parse the HTML text into a DOM structure + const parser = new DOMParser(); + const doc = parser.parseFromString(htmlText, 'text/html'); + + // Extract all
  • elements + const listItems = doc.querySelectorAll('li'); + + // Get the text content of each
  • element + const itemNames = Array.from(listItems).map(li => li.textContent.trim()); + + return itemNames; + } catch (error) { + console.error('Error fetching or parsing HTML:', error); + return []; + } + } + + async displayText(text: string, size: string) { + //(d,F,[number]) == s, m, or l + await this.sendWebsocketCommand(command.display, "F", size); await this.sendWebsocketCommand(command.display, "t", text); } + + /** * NOTE: Consider making private * @param command diff --git a/extensions/src/doodlebot/UI.svelte b/extensions/src/doodlebot/UI.svelte new file mode 100644 index 000000000..16e9a9fea --- /dev/null +++ b/extensions/src/doodlebot/UI.svelte @@ -0,0 +1,101 @@ + + +
    +

    Hello, world!

    +

    I am {extension.name}.

    + +
    + + diff --git a/extensions/src/doodlebot/enums.ts b/extensions/src/doodlebot/enums.ts index a7016bbc1..19542ea44 100644 --- a/extensions/src/doodlebot/enums.ts +++ b/extensions/src/doodlebot/enums.ts @@ -30,6 +30,7 @@ export const command = { display: "d", pen: "u", network: "g", + speaker: "s" } as const; export type CommandKey = keyof typeof command; diff --git a/extensions/src/doodlebot/index.ts b/extensions/src/doodlebot/index.ts index f4ce9c186..5489088b2 100644 --- a/extensions/src/doodlebot/index.ts +++ b/extensions/src/doodlebot/index.ts @@ -1,7 +1,8 @@ -import { Environment, ExtensionMenuDisplayDetails, extension, block, buttonBlock } from "$common"; +import { Environment, ExtensionMenuDisplayDetails, extension, block, buttonBlock, scratch, BlockUtilityWithID } from "$common"; import { DisplayKey, displayKeys, command, type Command, SensorKey, sensorKeys } from "./enums"; import Doodlebot from "./Doodlebot"; import { splitArgsString } from "./utils"; +import CustomArgument from './CustomArgument.svelte'; import EventEmitter from "events"; import { categoryByGesture, classes, emojiByGesture, gestureDetection, gestureMenuItems, gestures, objectDetection } from "./detection"; @@ -35,7 +36,10 @@ const looper = (action: () => Promise, profileMarker?: string) => { return controller; } -export default class DoodlebotBlocks extends extension(details, "ui", "indicators", "video", "drawable") { +export var imageFiles = []; +export var soundFiles: string[] = []; + +export default class DoodlebotBlocks extends extension(details, "ui", "customArguments", "indicators", "video", "drawable") { doodlebot: Doodlebot; private indicator: Promise<{ close(): void; }>; @@ -56,18 +60,34 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator imageStream: HTMLImageElement; videoDrawable: ReturnType; + ip: string; + init(env: Environment) { this.openUI("Connect"); this.setIndicator("disconnected"); + console.log(env); + + soundFiles = ["test"]; + imageFiles = ["test"]; + // idea: set up polling mechanism to try and disable unused sensors // idea: set up polling mechanism to destroy gesture recognition loop } - setDoodlebot(doodlebot: Doodlebot) { + async setIP(ip: string) { + this.ip = ip; + + } + + async setDoodlebot(doodlebot: Doodlebot) { this.doodlebot = doodlebot; this.setIndicator("connected"); + imageFiles = await doodlebot.findImageFiles(); + soundFiles = await doodlebot.findSoundFiles(); + console.log("SETTING"); + console.log(soundFiles); } async setIndicator(status: "connected" | "disconnected") { @@ -102,16 +122,17 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator @block({ type: "command", - text: (direction, steps) => `drive ${direction} for ${steps} steps`, + text: (direction, steps, speed) => `drive ${direction} for ${steps} steps at speed ${speed}`, args: [ { type: "string", options: ["forward", "backward", "left", "right"], defaultValue: "forward" }, - { type: "number", defaultValue: 2000 } + { type: "number", defaultValue: 2000 }, + { type: "number", options: [1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000], defaultValue: 2000 } ] }) - async drive(direction: "left" | "right" | "forward" | "backward", steps: number) { + async drive(direction: "left" | "right" | "forward" | "backward", steps: number, speed: number) { const leftSteps = direction == "left" || direction == "backward" ? -steps : steps; const rightSteps = direction == "right" || direction == "backward" ? -steps : steps; - const stepsPerSecond = 2000; + const stepsPerSecond = speed; await this.doodlebot?.motorCommand( "steps", @@ -237,11 +258,12 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator @block({ type: "command", - text: (text: string) => `display text ${text}`, - arg: { type: "string", defaultValue: "hello world!" } + text: (text: string, size: string) => `display text ${text} with size ${size}`, + args: [{ type: "string", defaultValue: "hello world!" }, + { type: "string", options: ["s", "m", "l"], defaultValue: "m" }] }) - async setText(text: string) { - await this.doodlebot?.displayText(text); + async setText(text: string, size: string) { + await this.doodlebot?.displayText(text, size); } @block({ @@ -261,6 +283,30 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator await this.doodlebot?.sendWebsocketCommand("m", sound) } + @block((self) => ({ + type: "command", + text: (sound) => `play sound file ${sound}`, + // arg: self.makeCustomArgument({ + // component: CustomArgument, + // initial: { value: "File", text: "File" } + // }) + arg: { type: "string", options: () => soundFiles } + })) + // @(scratch.command( + // (self, tag) => tag`play sound file ${{ type: "string", options: self.soundFiles }}` + // )) + async playSoundFile(sound: string, util: BlockUtilityWithID) { + await this.doodlebot?.sendWebsocketCommand("m", sound) + const { target } = util; + console.log(target); + if (target.sprite) { + console.log(target.sprite); + let soundArray = target.sprite.sounds[0].asset.data; + console.log(soundArray); + this.doodlebot.sendAudioData(soundArray); + } + } + @block({ type: "command", text: (transparency) => `display video with ${transparency}% transparency`, @@ -345,6 +391,48 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator } + async downloadAndUploadFile(fileUrl: string, uploadEndpoint: string): Promise { + try { + // Step 1: Download the file from the URL + const response = await fetch(fileUrl); + if (!response.ok) { + throw new Error(`Failed to fetch file from URL: ${response.statusText}`); + } + + // Convert response to a Blob + const fileBlob = await response.blob(); + + // Step 2: Prepare FormData for upload + const formData = new FormData(); + formData.append('file', fileBlob, 'uploaded-file'); // Optional: 'uploaded-file' is the filename + + // Step 3: Upload the file to the server + const uploadResponse = await fetch(uploadEndpoint, { + method: 'POST', + body: formData, + headers: { + // 'Content-Type': 'multipart/form-data', // No need to set Content-Type manually + }, + }); + + if (!uploadResponse.ok) { + throw new Error(`Failed to upload file: ${uploadResponse.statusText}`); + } + + const result = await uploadResponse.json(); + console.log('File uploaded successfully:', result); + } catch (error) { + console.error('Error:', error); + } + } + + async setArrays() { + imageFiles = await this.doodlebot.findImageFiles(); + soundFiles = await this.doodlebot.findSoundFiles(); + soundFiles = await this.doodlebot.findSoundFiles(); + } + + @block({ type: "reporter", text: "get IP address" @@ -353,6 +441,23 @@ export default class DoodlebotBlocks extends extension(details, "ui", "indicator return this.doodlebot?.getIPAddress(); } + @block({ + type: "command", + text: "Upload files" + }) + async uploadFiles() { + this.openUI("UI"); + } + + @block({ + type: "command", + text: "Find files" + }) + async findFiles() { + let response = await this.doodlebot?.fetchAndExtractList("http://192.168.41.121:8080/sounds/"); + console.log(response); + } + @block({ type: "command", text: (_command, args, protocol) => `send (${_command}, ${args}) over ${protocol}`, From 1ae073bdc6cbb0dc8d60c8fc6c848d164813ba19 Mon Sep 17 00:00:00 2001 From: ymayarajan3 Date: Thu, 1 Aug 2024 15:25:37 -0400 Subject: [PATCH 02/11] separate code --- extensions/src/doodlebot/index.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/extensions/src/doodlebot/index.ts b/extensions/src/doodlebot/index.ts index 5489088b2..f9ef7209c 100644 --- a/extensions/src/doodlebot/index.ts +++ b/extensions/src/doodlebot/index.ts @@ -297,6 +297,21 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg // )) async playSoundFile(sound: string, util: BlockUtilityWithID) { await this.doodlebot?.sendWebsocketCommand("m", sound) + } + + @block((self) => ({ + type: "command", + text: (sound) => `play asset file ${sound}`, + // arg: self.makeCustomArgument({ + // component: CustomArgument, + // initial: { value: "File", text: "File" } + // }) + arg: { type: "string", options: () => soundFiles } + })) + // @(scratch.command( + // (self, tag) => tag`play sound file ${{ type: "string", options: self.soundFiles }}` + // )) + async playAssetFile(sound: string, util: BlockUtilityWithID) { const { target } = util; console.log(target); if (target.sprite) { @@ -429,7 +444,6 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg async setArrays() { imageFiles = await this.doodlebot.findImageFiles(); soundFiles = await this.doodlebot.findSoundFiles(); - soundFiles = await this.doodlebot.findSoundFiles(); } From 7cc4289de7abd37c98706b121425f3d79f74cf50 Mon Sep 17 00:00:00 2001 From: ymayarajan3 Date: Thu, 1 Aug 2024 23:06:46 -0400 Subject: [PATCH 03/11] block updates --- extensions/src/doodlebot/Doodlebot.ts | 24 ++-- extensions/src/doodlebot/index.ts | 161 ++++++++++++-------------- 2 files changed, 83 insertions(+), 102 deletions(-) diff --git a/extensions/src/doodlebot/Doodlebot.ts b/extensions/src/doodlebot/Doodlebot.ts index 756c148df..c9a0c6404 100644 --- a/extensions/src/doodlebot/Doodlebot.ts +++ b/extensions/src/doodlebot/Doodlebot.ts @@ -632,6 +632,11 @@ export default class Doodlebot { }); } + getStoredIPAddress() { + if (!this.connection) { return "" } + return this.connection.ip; + } + parseWavHeader(uint8Array) { const dataView = new DataView(uint8Array.buffer); @@ -679,9 +684,6 @@ export default class Doodlebot { ws.onopen = () => { console.log('WebSocket connection opened'); - - let offset = 0; - let { sampleWidth, channels, rate } = this.parseWavHeader(uint8Array); let first = "(1," + String(sampleWidth) + "," + String(channels) + "," + String(rate) + ")"; console.log(first); @@ -695,21 +697,16 @@ export default class Doodlebot { return; } - // Calculate end position of the current chunk - //const end = Math.min(offset + CHUNK_SIZE, uint8Array.length); const chunk = chunks[i]; - console.log("sending"); const binaryString = Array.from(chunk).map((byte: any) => String.fromCharCode(byte)).join('');; const base64Data = btoa(binaryString); const jsonData = JSON.stringify({ audio_data: base64Data }); - console.log(jsonData); ws.send(jsonData); i = i + 1; sendNextChunk(); } - // Start sending chunks sendNextChunk(); }; @@ -717,11 +714,8 @@ export default class Doodlebot { console.error('WebSocket error:', error); }; - ws.onmessage = (event) => { - const response = JSON.parse(event.data); - if (response.type === 'ack' && response.status === 'received') { - console.log('Message successfully received by server'); - } + ws.onmessage = (message) => { + console.log(message); } ws.onclose = () => { @@ -837,6 +831,10 @@ export default class Doodlebot { await this.sendWebsocketCommand(command.display, value); } + async displayFile(file: string) { + await this.sendWebsocketCommand(command.display, file); + } + // Function to fetch and parse HTML template async fetchAndExtractList(endpoint) { try { diff --git a/extensions/src/doodlebot/index.ts b/extensions/src/doodlebot/index.ts index f9ef7209c..562e0e82b 100644 --- a/extensions/src/doodlebot/index.ts +++ b/extensions/src/doodlebot/index.ts @@ -3,6 +3,8 @@ import { DisplayKey, displayKeys, command, type Command, SensorKey, sensorKeys } import Doodlebot from "./Doodlebot"; import { splitArgsString } from "./utils"; import CustomArgument from './CustomArgument.svelte'; +import SoundArgument from './SoundArgument.svelte'; +import ImageArgument from './ImageArgument.svelte'; import EventEmitter from "events"; import { categoryByGesture, classes, emojiByGesture, gestureDetection, gestureMenuItems, gestures, objectDetection } from "./detection"; @@ -61,16 +63,27 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg imageStream: HTMLImageElement; videoDrawable: ReturnType; ip: string; + soundDictionary; init(env: Environment) { this.openUI("Connect"); this.setIndicator("disconnected"); + this.soundDictionary = {}; + + for (const target of env.runtime.targets) { + this.soundDictionary[target.id] = {}; + if (target.sprite) { + for (const sound of target.sprite.sounds) { + if (sound.asset.dataFormat == "wav") { + this.soundDictionary[target.id][sound.name] = sound.asset.data; + } + } + } + } - console.log(env); - - soundFiles = ["test"]; - imageFiles = ["test"]; + soundFiles = ["File"]; + imageFiles = ["File"]; // idea: set up polling mechanism to try and disable unused sensors // idea: set up polling mechanism to destroy gesture recognition loop @@ -81,6 +94,10 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg } + getCurrentSounds(id): string[] { + return Object.keys(this.soundDictionary[id]); + } + async setDoodlebot(doodlebot: Doodlebot) { this.doodlebot = doodlebot; this.setIndicator("connected"); @@ -90,6 +107,10 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg console.log(soundFiles); } + async getIPAddress() { + return this.doodlebot?.getStoredIPAddress(); + } + async setIndicator(status: "connected" | "disconnected") { if (this.indicator) (await this.indicator)?.close(); this.indicator = status == "connected" @@ -247,13 +268,18 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg await this.doodlebot?.disableSensor(sensor); } - @block({ + @block((self) => ({ type: "command", - text: (type: DisplayKey) => `display ${type}`, - arg: { type: "string", options: displayKeys.filter(key => key !== "clear"), defaultValue: "happy" } - }) - async setDisplay(display: DisplayKey) { - await this.doodlebot?.display(display); + text: (type: DisplayKey | string) => `display ${type}`, + arg: { type: "string", options: () => displayKeys.filter(key => key !== "clear").concat(imageFiles), defaultValue: "happy" } + })) + async setDisplay(display: DisplayKey | string) { + if (imageFiles.includes(display)) { + await this.doodlebot?.displayFile(display); + } else { + await this.doodlebot?.display(display as DisplayKey); + } + } @block({ @@ -286,42 +312,24 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg @block((self) => ({ type: "command", text: (sound) => `play sound file ${sound}`, - // arg: self.makeCustomArgument({ - // component: CustomArgument, - // initial: { value: "File", text: "File" } - // }) - arg: { type: "string", options: () => soundFiles } + arg: { + type: "string", options: () => soundFiles.concat(self.getCurrentSounds(self.runtime._editingTarget.id)) + } })) - // @(scratch.command( - // (self, tag) => tag`play sound file ${{ type: "string", options: self.soundFiles }}` - // )) async playSoundFile(sound: string, util: BlockUtilityWithID) { - await this.doodlebot?.sendWebsocketCommand("m", sound) - } - - @block((self) => ({ - type: "command", - text: (sound) => `play asset file ${sound}`, - // arg: self.makeCustomArgument({ - // component: CustomArgument, - // initial: { value: "File", text: "File" } - // }) - arg: { type: "string", options: () => soundFiles } - })) - // @(scratch.command( - // (self, tag) => tag`play sound file ${{ type: "string", options: self.soundFiles }}` - // )) - async playAssetFile(sound: string, util: BlockUtilityWithID) { - const { target } = util; - console.log(target); - if (target.sprite) { - console.log(target.sprite); - let soundArray = target.sprite.sounds[0].asset.data; + let currentId = this.runtime._editingTarget.id; + let costumeSounds = this.getCurrentSounds(currentId); + if (costumeSounds.includes(sound)) { + let soundArray = this.soundDictionary[currentId][sound]; console.log(soundArray); - this.doodlebot.sendAudioData(soundArray); + await this.doodlebot.sendAudioData(soundArray); + } else { + await this.doodlebot?.sendWebsocketCommand("m", sound) } + } + @block({ type: "command", text: (transparency) => `display video with ${transparency}% transparency`, @@ -405,45 +413,10 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg await new Promise((resolve) => setTimeout(resolve, audioDuration * 1000)); } - - async downloadAndUploadFile(fileUrl: string, uploadEndpoint: string): Promise { - try { - // Step 1: Download the file from the URL - const response = await fetch(fileUrl); - if (!response.ok) { - throw new Error(`Failed to fetch file from URL: ${response.statusText}`); - } - - // Convert response to a Blob - const fileBlob = await response.blob(); - - // Step 2: Prepare FormData for upload - const formData = new FormData(); - formData.append('file', fileBlob, 'uploaded-file'); // Optional: 'uploaded-file' is the filename - - // Step 3: Upload the file to the server - const uploadResponse = await fetch(uploadEndpoint, { - method: 'POST', - body: formData, - headers: { - // 'Content-Type': 'multipart/form-data', // No need to set Content-Type manually - }, - }); - - if (!uploadResponse.ok) { - throw new Error(`Failed to upload file: ${uploadResponse.statusText}`); - } - - const result = await uploadResponse.json(); - console.log('File uploaded successfully:', result); - } catch (error) { - console.error('Error:', error); - } - } - async setArrays() { imageFiles = await this.doodlebot.findImageFiles(); soundFiles = await this.doodlebot.findSoundFiles(); + console.log("SETTING"); } @@ -455,23 +428,33 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg return this.doodlebot?.getIPAddress(); } - @block({ - type: "command", - text: "Upload files" - }) - async uploadFiles() { - this.openUI("UI"); + // @block({ + // type: "command", + // text: "Upload files" + // }) + // async uploadFiles() { + // this.openUI("UI"); + // } + + @(scratch.command((self, $) => $`Upload sound file ${self.makeCustomArgument({ component: SoundArgument, initial: { value: "File", text: "File" } })}`)) + uploadSoundFile(test: string) { + } - @block({ - type: "command", - text: "Find files" - }) - async findFiles() { - let response = await this.doodlebot?.fetchAndExtractList("http://192.168.41.121:8080/sounds/"); - console.log(response); + @(scratch.command((self, $) => $`Upload image file ${self.makeCustomArgument({ component: ImageArgument, initial: { value: "File", text: "File" } })}`)) + uploadImageFile(test: string) { + } + // @block({ + // type: "command", + // text: "Find files" + // }) + // async findFiles() { + // let response = await this.doodlebot?.fetchAndExtractList("http://192.168.41.121:8080/sounds/"); + // console.log(response); + // } + @block({ type: "command", text: (_command, args, protocol) => `send (${_command}, ${args}) over ${protocol}`, From c559c46db3dff54ed138da395371852f1f064c37 Mon Sep 17 00:00:00 2001 From: ymayarajan3 Date: Thu, 1 Aug 2024 23:13:38 -0400 Subject: [PATCH 04/11] argument files --- extensions/src/doodlebot/ImageArgument.svelte | 73 +++++++++++++++++++ extensions/src/doodlebot/SoundArgument.svelte | 73 +++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 extensions/src/doodlebot/ImageArgument.svelte create mode 100644 extensions/src/doodlebot/SoundArgument.svelte diff --git a/extensions/src/doodlebot/ImageArgument.svelte b/extensions/src/doodlebot/ImageArgument.svelte new file mode 100644 index 000000000..3785dc036 --- /dev/null +++ b/extensions/src/doodlebot/ImageArgument.svelte @@ -0,0 +1,73 @@ + + + + + diff --git a/extensions/src/doodlebot/SoundArgument.svelte b/extensions/src/doodlebot/SoundArgument.svelte new file mode 100644 index 000000000..964fd45ad --- /dev/null +++ b/extensions/src/doodlebot/SoundArgument.svelte @@ -0,0 +1,73 @@ + + + + + From fec2e79b7fa2e6b0eefda5bf7cc0c48dee22327f Mon Sep 17 00:00:00 2001 From: ymayarajan3 Date: Fri, 2 Aug 2024 13:22:56 -0400 Subject: [PATCH 05/11] moving upload to execution --- extensions/src/doodlebot/Doodlebot.ts | 7 --- ...undArgument.svelte => FileArgument.svelte} | 31 ++-------- extensions/src/doodlebot/index.ts | 57 ++++++++++++++++--- 3 files changed, 55 insertions(+), 40 deletions(-) rename extensions/src/doodlebot/{SoundArgument.svelte => FileArgument.svelte} (63%) diff --git a/extensions/src/doodlebot/Doodlebot.ts b/extensions/src/doodlebot/Doodlebot.ts index c9a0c6404..284eb7175 100644 --- a/extensions/src/doodlebot/Doodlebot.ts +++ b/extensions/src/doodlebot/Doodlebot.ts @@ -430,7 +430,6 @@ export default class Doodlebot { await new Promise(resolve => setTimeout(resolve, 100)); } let endpoint = "http://" + this.connection.ip + ":8080/images/" - console.log(endpoint); let uploadedImages = await this.fetchAndExtractList(endpoint); return uploadedImages.filter(item => !this.imageFiles.includes(item)); } @@ -442,9 +441,6 @@ export default class Doodlebot { if (!this.connection) return []; let endpoint = "http://" + this.connection.ip + ":8080/sounds/" let uploadedSounds = await this.fetchAndExtractList(endpoint); - console.log("uploaded"); - console.log(uploadedSounds); - console.log(uploadedSounds.filter(item => !this.soundFiles.includes(item))); return uploadedSounds.filter(item => !this.soundFiles.includes(item)); } @@ -847,9 +843,6 @@ export default class Doodlebot { // Get the HTML text const htmlText = await response.text(); - console.log(response); - console.log(htmlText); - // Parse the HTML text into a DOM structure const parser = new DOMParser(); const doc = parser.parseFromString(htmlText, 'text/html'); diff --git a/extensions/src/doodlebot/SoundArgument.svelte b/extensions/src/doodlebot/FileArgument.svelte similarity index 63% rename from extensions/src/doodlebot/SoundArgument.svelte rename to extensions/src/doodlebot/FileArgument.svelte index 964fd45ad..e5351a4a8 100644 --- a/extensions/src/doodlebot/SoundArgument.svelte +++ b/extensions/src/doodlebot/FileArgument.svelte @@ -15,7 +15,7 @@ * The second generic argumen is the name of the block function this argument belongs to. * The third generic argumen is the index of the argument (i.e. is the functions 2nd argument? Then it's index would be 1) */ - type Value = ParameterOf; + type Value = ParameterOf; // svelte-ignore unused-export-let export let setter: ArgumentEntrySetter; @@ -26,7 +26,7 @@ // svelte-ignore unused-export-let export let extension: Extension; let value = current.value; - $: text = value; + $: text = ""; const invoke: ReactiveInvoke = (functionName, ...args) => reactiveInvoke((extension = extension), functionName, args); @@ -38,30 +38,9 @@ const input = event.target as HTMLInputElement; if (input.files && input.files[0]) { const file = input.files[0]; - value = file.name; - let ip = await extension.getIPAddress(); - const uploadEndpoint = "http://" + ip + ":8080/sounds_upload"; - - try { - const formData = new FormData(); - formData.append("file", file); - - const response = await fetch(uploadEndpoint, { - method: "POST", - body: formData, - }); - - console.log(response); - - if (!response.ok) { - throw new Error(`Failed to upload file: ${response.statusText}`); - } - - console.log("File uploaded successfully"); - invoke("setArrays"); - } catch (error) { - console.error("Error:", error); - } + const blobUrl = URL.createObjectURL(file); + value = file.name + "---name---" + blobUrl; + text = file.name; } }); }); diff --git a/extensions/src/doodlebot/index.ts b/extensions/src/doodlebot/index.ts index 562e0e82b..2c5d01f10 100644 --- a/extensions/src/doodlebot/index.ts +++ b/extensions/src/doodlebot/index.ts @@ -2,9 +2,7 @@ import { Environment, ExtensionMenuDisplayDetails, extension, block, buttonBlock import { DisplayKey, displayKeys, command, type Command, SensorKey, sensorKeys } from "./enums"; import Doodlebot from "./Doodlebot"; import { splitArgsString } from "./utils"; -import CustomArgument from './CustomArgument.svelte'; -import SoundArgument from './SoundArgument.svelte'; -import ImageArgument from './ImageArgument.svelte'; +import FileArgument from './FileArgument.svelte'; import EventEmitter from "events"; import { categoryByGesture, classes, emojiByGesture, gestureDetection, gestureMenuItems, gestures, objectDetection } from "./detection"; @@ -73,6 +71,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg for (const target of env.runtime.targets) { this.soundDictionary[target.id] = {}; + console.log(target); if (target.sprite) { for (const sound of target.sprite.sounds) { if (sound.asset.dataFormat == "wav") { @@ -436,14 +435,58 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg // this.openUI("UI"); // } - @(scratch.command((self, $) => $`Upload sound file ${self.makeCustomArgument({ component: SoundArgument, initial: { value: "File", text: "File" } })}`)) - uploadSoundFile(test: string) { + async uploadFile(type: string, blobURL: string) { + const ip = await this.getIPAddress(); + let uploadEndpoint; + if (type == "sound") { + uploadEndpoint = "http://" + ip + ":8080/sounds_upload"; + } else { + uploadEndpoint = "http://" + ip + ":8080/img_upload"; + } + + try { + const components = blobURL.split("---name---"); + console.log("COMPONENTS"); + console.log(components); + const response1 = await fetch(components[1]); + if (!response1.ok) { + throw new Error(`Failed to fetch Blob from URL: ${blobURL}`); + } + const blob = await response1.blob(); + // Convert Blob to File + const file = new File([blob], components[0], { type: blob.type }); + const formData = new FormData(); + formData.append("file", file); + + console.log("file"); + console.log(file); + + const response2 = await fetch(uploadEndpoint, { + method: "POST", + body: formData, + }); + console.log(response2); + + if (!response2.ok) { + throw new Error(`Failed to upload file: ${response2.statusText}`); + } + + console.log("File uploaded successfully"); + this.setArrays(); + } catch (error) { + console.error("Error:", error); + } } - @(scratch.command((self, $) => $`Upload image file ${self.makeCustomArgument({ component: ImageArgument, initial: { value: "File", text: "File" } })}`)) - uploadImageFile(test: string) { + @(scratch.command((self, $) => $`Upload sound file ${self.makeCustomArgument({ component: FileArgument, initial: { value: "", text: "File" } })}`)) + async uploadSoundFile(test: string) { + await this.uploadFile("sound", test); + } + @(scratch.command((self, $) => $`Upload image file ${self.makeCustomArgument({ component: FileArgument, initial: { value: "", text: "File" } })}`)) + async uploadImageFile(test: string) { + await this.uploadFile("image", test); } // @block({ From ffc818da2e09bd5bfc79586a8f4aa3c5b5c877fe Mon Sep 17 00:00:00 2001 From: ymayarajan3 Date: Fri, 2 Aug 2024 14:49:48 -0400 Subject: [PATCH 06/11] adding costumes --- extensions/src/doodlebot/index.ts | 63 ++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/extensions/src/doodlebot/index.ts b/extensions/src/doodlebot/index.ts index 2c5d01f10..80fc08616 100644 --- a/extensions/src/doodlebot/index.ts +++ b/extensions/src/doodlebot/index.ts @@ -62,15 +62,18 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg videoDrawable: ReturnType; ip: string; soundDictionary; + costumeDictionary; - init(env: Environment) { + async init(env: Environment) { this.openUI("Connect"); this.setIndicator("disconnected"); this.soundDictionary = {}; + this.costumeDictionary = {}; for (const target of env.runtime.targets) { this.soundDictionary[target.id] = {}; + this.costumeDictionary[target.id] = {}; console.log(target); if (target.sprite) { for (const sound of target.sprite.sounds) { @@ -78,6 +81,15 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg this.soundDictionary[target.id][sound.name] = sound.asset.data; } } + for (const costume of target.sprite.costumes) { + await this.convertSvgUint8ArrayToPng(costume.asset.data, costume.size[0], costume.size[1]) + .then((pngBlob: Blob) => { + const url = URL.createObjectURL(pngBlob) + this.costumeDictionary[target.id][costume.name] = "costume9999.png---name---" + url; + }) + + + } } } @@ -93,6 +105,49 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg } + async convertSvgUint8ArrayToPng(uint8Array, width, height) { + return new Promise((resolve, reject) => { + // Convert Uint8Array to a string + const svgString = new TextDecoder().decode(uint8Array); + + // Create an SVG Blob + const svgBlob = new Blob([svgString], { type: 'image/svg+xml;charset=utf-8' }); + const url = URL.createObjectURL(svgBlob); + + // Create an Image element + const img = new Image(); + img.onload = () => { + // Create a canvas element + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext('2d'); + + // Draw the image on the canvas + ctx.drawImage(img, 0, 0, width, height); + + // Convert the canvas to PNG data URL + const pngDataUrl = canvas.toDataURL('image/png'); + + // Convert the data URL to a Blob + fetch(pngDataUrl) + .then(res => res.blob()) + .then(blob => { + // Clean up + URL.revokeObjectURL(url); + resolve(blob); + }) + .catch(err => { + URL.revokeObjectURL(url); + reject(err); + }); + }; + + img.onerror = reject; + img.src = url; + }); + } + getCurrentSounds(id): string[] { return Object.keys(this.soundDictionary[id]); } @@ -489,6 +544,12 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg await this.uploadFile("image", test); } + @(scratch.command((self, $) => $`Upload costume ${{ type: "string", options: Object.keys(self.costumeDictionary[self.runtime._editingTarget.id]) }}`)) + async uploadCostume(test: string) { + await this.uploadFile("image", this.costumeDictionary[this.runtime._editingTarget.id][test]); + await this.setArrays(); + } + // @block({ // type: "command", // text: "Find files" From 00aaad0891ba232c1591b587409d4c061b4a7463 Mon Sep 17 00:00:00 2001 From: ymayarajan3 Date: Fri, 2 Aug 2024 14:53:04 -0400 Subject: [PATCH 07/11] adding costume display --- extensions/src/doodlebot/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extensions/src/doodlebot/index.ts b/extensions/src/doodlebot/index.ts index 80fc08616..0cdc8626c 100644 --- a/extensions/src/doodlebot/index.ts +++ b/extensions/src/doodlebot/index.ts @@ -544,10 +544,11 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg await this.uploadFile("image", test); } - @(scratch.command((self, $) => $`Upload costume ${{ type: "string", options: Object.keys(self.costumeDictionary[self.runtime._editingTarget.id]) }}`)) + @(scratch.command((self, $) => $`Display costume ${{ type: "string", options: Object.keys(self.costumeDictionary[self.runtime._editingTarget.id]) }}`)) async uploadCostume(test: string) { await this.uploadFile("image", this.costumeDictionary[this.runtime._editingTarget.id][test]); await this.setArrays(); + await this.doodlebot.displayFile("costume9999.png"); } // @block({ From a87cc014c90db3eca51a75fc71916891b80159ba Mon Sep 17 00:00:00 2001 From: ymayarajan3 Date: Fri, 2 Aug 2024 15:10:43 -0400 Subject: [PATCH 08/11] multiple costume formats --- extensions/src/doodlebot/index.ts | 42 ++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/extensions/src/doodlebot/index.ts b/extensions/src/doodlebot/index.ts index 0cdc8626c..c27776f3c 100644 --- a/extensions/src/doodlebot/index.ts +++ b/extensions/src/doodlebot/index.ts @@ -71,7 +71,17 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg this.soundDictionary = {}; this.costumeDictionary = {}; - for (const target of env.runtime.targets) { + await this.setDictionaries(); + + soundFiles = ["File"]; + imageFiles = ["File"]; + + // idea: set up polling mechanism to try and disable unused sensors + // idea: set up polling mechanism to destroy gesture recognition loop + } + + async setDictionaries() { + for (const target of this.runtime.targets) { this.soundDictionary[target.id] = {}; this.costumeDictionary[target.id] = {}; console.log(target); @@ -82,22 +92,21 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg } } for (const costume of target.sprite.costumes) { - await this.convertSvgUint8ArrayToPng(costume.asset.data, costume.size[0], costume.size[1]) - .then((pngBlob: Blob) => { - const url = URL.createObjectURL(pngBlob) - this.costumeDictionary[target.id][costume.name] = "costume9999.png---name---" + url; - }) - + if (costume.asset.dataFormat == "svg") { + await this.convertSvgUint8ArrayToPng(costume.asset.data, costume.size[0], costume.size[1]) + .then((pngBlob: Blob) => { + const url = URL.createObjectURL(pngBlob) + this.costumeDictionary[target.id][costume.name] = "costume9999.png---name---" + url; + }) + } else if (costume.asset.dataFormat == "png") { + const blob = new Blob([costume.asset.data], { type: 'image/png' }); + const url = URL.createObjectURL(blob) + this.costumeDictionary[target.id][costume.name] = "costume9999.png---name---" + url; + } } } } - - soundFiles = ["File"]; - imageFiles = ["File"]; - - // idea: set up polling mechanism to try and disable unused sensors - // idea: set up polling mechanism to destroy gesture recognition loop } async setIP(ip: string) { @@ -544,7 +553,12 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg await this.uploadFile("image", test); } - @(scratch.command((self, $) => $`Display costume ${{ type: "string", options: Object.keys(self.costumeDictionary[self.runtime._editingTarget.id]) }}`)) + @(scratch.command((self, $) => $`Display costume ${{ + type: "string", options: () => { + self.setDictionaries(); + return Object.keys(self.costumeDictionary[self.runtime._editingTarget.id]) + } + }}`)) async uploadCostume(test: string) { await this.uploadFile("image", this.costumeDictionary[this.runtime._editingTarget.id][test]); await this.setArrays(); From db77a1946afbc4c1f7b971818bda12e09fa70e74 Mon Sep 17 00:00:00 2001 From: ymayarajan3 Date: Sat, 3 Aug 2024 19:06:17 -0400 Subject: [PATCH 09/11] finalize costume code --- extensions/src/doodlebot/index.ts | 49 +++++++++++++------------------ 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/extensions/src/doodlebot/index.ts b/extensions/src/doodlebot/index.ts index c27776f3c..339715129 100644 --- a/extensions/src/doodlebot/index.ts +++ b/extensions/src/doodlebot/index.ts @@ -62,7 +62,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg videoDrawable: ReturnType; ip: string; soundDictionary; - costumeDictionary; + costumeDictionary: any; async init(env: Environment) { @@ -71,6 +71,10 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg this.soundDictionary = {}; this.costumeDictionary = {}; + env.runtime.on("TARGETS_UPDATE", async () => { + await this.setDictionaries(); + }) + await this.setDictionaries(); soundFiles = ["File"]; @@ -84,7 +88,6 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg for (const target of this.runtime.targets) { this.soundDictionary[target.id] = {}; this.costumeDictionary[target.id] = {}; - console.log(target); if (target.sprite) { for (const sound of target.sprite.sounds) { if (sound.asset.dataFormat == "wav") { @@ -92,16 +95,17 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg } } for (const costume of target.sprite.costumes) { + let id = "Costume: " + costume.name; if (costume.asset.dataFormat == "svg") { await this.convertSvgUint8ArrayToPng(costume.asset.data, costume.size[0], costume.size[1]) .then((pngBlob: Blob) => { const url = URL.createObjectURL(pngBlob) - this.costumeDictionary[target.id][costume.name] = "costume9999.png---name---" + url; + this.costumeDictionary[target.id][id] = "costume9999.png---name---" + url; }) } else if (costume.asset.dataFormat == "png") { const blob = new Blob([costume.asset.data], { type: 'image/png' }); const url = URL.createObjectURL(blob) - this.costumeDictionary[target.id][costume.name] = "costume9999.png---name---" + url; + this.costumeDictionary[target.id][id] = "costume9999.png---name---" + url; } } @@ -334,10 +338,20 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg @block((self) => ({ type: "command", text: (type: DisplayKey | string) => `display ${type}`, - arg: { type: "string", options: () => displayKeys.filter(key => key !== "clear").concat(imageFiles), defaultValue: "happy" } + arg: { + type: "string", options: () => { + self.setDictionaries(); + return displayKeys.filter(key => key !== "clear").concat(imageFiles).concat(Object.keys(self.costumeDictionary[self.runtime._editingTarget.id]) as any[]).filter((item: string) => item != "costume9999.png") + }, defaultValue: "happy" + } })) async setDisplay(display: DisplayKey | string) { - if (imageFiles.includes(display)) { + let costumeNames = Object.keys(this.costumeDictionary[this.runtime._editingTarget.id]); + if (costumeNames.includes(display)) { + await this.uploadFile("image", this.costumeDictionary[this.runtime._editingTarget.id][display]); + await this.setArrays(); + await this.doodlebot.displayFile("costume9999.png"); + } else if (imageFiles.includes(display)) { await this.doodlebot?.displayFile(display); } else { await this.doodlebot?.display(display as DisplayKey); @@ -374,7 +388,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg @block((self) => ({ type: "command", - text: (sound) => `play sound file ${sound}`, + text: (sound) => `play sound ${sound}`, arg: { type: "string", options: () => soundFiles.concat(self.getCurrentSounds(self.runtime._editingTarget.id)) } @@ -553,27 +567,6 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg await this.uploadFile("image", test); } - @(scratch.command((self, $) => $`Display costume ${{ - type: "string", options: () => { - self.setDictionaries(); - return Object.keys(self.costumeDictionary[self.runtime._editingTarget.id]) - } - }}`)) - async uploadCostume(test: string) { - await this.uploadFile("image", this.costumeDictionary[this.runtime._editingTarget.id][test]); - await this.setArrays(); - await this.doodlebot.displayFile("costume9999.png"); - } - - // @block({ - // type: "command", - // text: "Find files" - // }) - // async findFiles() { - // let response = await this.doodlebot?.fetchAndExtractList("http://192.168.41.121:8080/sounds/"); - // console.log(response); - // } - @block({ type: "command", text: (_command, args, protocol) => `send (${_command}, ${args}) over ${protocol}`, From ced4fbbd11e639afcb182db216475e8a63739e04 Mon Sep 17 00:00:00 2001 From: ymayarajan3 Date: Tue, 6 Aug 2024 21:08:19 -0400 Subject: [PATCH 10/11] sound/image error catching --- extensions/src/doodlebot/ArrayError.svelte | 40 ++++++++++++++++++++++ extensions/src/doodlebot/Doodlebot.ts | 3 +- extensions/src/doodlebot/index.ts | 8 +++-- 3 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 extensions/src/doodlebot/ArrayError.svelte diff --git a/extensions/src/doodlebot/ArrayError.svelte b/extensions/src/doodlebot/ArrayError.svelte new file mode 100644 index 000000000..497616f7c --- /dev/null +++ b/extensions/src/doodlebot/ArrayError.svelte @@ -0,0 +1,40 @@ + + +
    +

    Cannot load Doodlebot's sound/image server!

    +

    Please try reloading the page.

    +
    + + diff --git a/extensions/src/doodlebot/Doodlebot.ts b/extensions/src/doodlebot/Doodlebot.ts index 284eb7175..111f207c8 100644 --- a/extensions/src/doodlebot/Doodlebot.ts +++ b/extensions/src/doodlebot/Doodlebot.ts @@ -855,8 +855,7 @@ export default class Doodlebot { return itemNames; } catch (error) { - console.error('Error fetching or parsing HTML:', error); - return []; + throw new Error('Error fetching or parsing HTML:', error) } } diff --git a/extensions/src/doodlebot/index.ts b/extensions/src/doodlebot/index.ts index 339715129..480d1969a 100644 --- a/extensions/src/doodlebot/index.ts +++ b/extensions/src/doodlebot/index.ts @@ -168,8 +168,12 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg async setDoodlebot(doodlebot: Doodlebot) { this.doodlebot = doodlebot; this.setIndicator("connected"); - imageFiles = await doodlebot.findImageFiles(); - soundFiles = await doodlebot.findSoundFiles(); + try { + imageFiles = await doodlebot.findImageFiles(); + soundFiles = await doodlebot.findSoundFiles(); + } catch (e) { + this.openUI("ArrayError"); + } console.log("SETTING"); console.log(soundFiles); } From af4079f1c03eddc68fec77b731688b25f633a88d Mon Sep 17 00:00:00 2001 From: ymayarajan3 Date: Wed, 7 Aug 2024 00:49:45 -0400 Subject: [PATCH 11/11] text input --- extensions/src/doodlebot/FileArgument.svelte | 40 +++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/extensions/src/doodlebot/FileArgument.svelte b/extensions/src/doodlebot/FileArgument.svelte index e5351a4a8..589002f9c 100644 --- a/extensions/src/doodlebot/FileArgument.svelte +++ b/extensions/src/doodlebot/FileArgument.svelte @@ -39,14 +39,52 @@ if (input.files && input.files[0]) { const file = input.files[0]; const blobUrl = URL.createObjectURL(file); + // Set the file name to the text input and the full value + document.getElementById("textInput").value = file.name; value = file.name + "---name---" + blobUrl; text = file.name; } }); + + // Handle text input change + document.getElementById("textInput")?.addEventListener("input", (event) => { + const input = event.target as HTMLInputElement; + // Update the value with the current text input value and maintain the blob URL + const blobUrl = document.querySelector("#fileInput").files[0] + ? URL.createObjectURL(document.querySelector("#fileInput").files[0]) + : ""; + value = input.value + "---name---" + blobUrl; + text = input.value; + }); }); - +
    + + +