diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..510bf730d --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,30 @@ +{ + "postStartCommand": "pnpm config set store-dir /home/vscode/.local/share/pnpm/store && pnpm install", + "customizations": { + "vscode": { + "extensions": [ + "ms-vscode.vscode-typescript-next", + "esbenp.prettier-vscode", + "svelte.svelte-vscode" + ], + "settings": { + "[svelte]": { + "editor.defaultFormatter": "svelte.svelte-vscode" + }, + "[typescript]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "editor.formatOnSave": true, + "editor.tabSize": 2, + "svelte.enable-ts-plugin": true + } + } + }, + "features": { + "ghcr.io/devcontainers/features/node:latest": { + "version": "20" + }, + "ghcr.io/pmalacho-mit/devcontainer-features/git-subrepo:latest": {} + }, + "image": "mcr.microsoft.com/devcontainers/base:ubuntu-24.04" +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 06b67e472..c97711554 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +**/.pnpm-store .DS_Store ./**/.DS_Store diff --git a/extensions/src/doodlebot/Connect.svelte b/extensions/src/doodlebot/Connect.svelte index 92ff1f8f8..ef1fe2b95 100644 --- a/extensions/src/doodlebot/Connect.svelte +++ b/extensions/src/doodlebot/Connect.svelte @@ -1,7 +1,11 @@
- {#if error} -
- {error} + {#if connected} +

You're connected to doodlebot!

+
+ If you'd like to reconnect, or connect to a different device, you must + reload this page.
- {/if} - {#if bluetooth} -

How to connect to doodlebot

-

1. Set network credentials:

-

- SSID (Network Name): - -

-

- Password: - -

+ +
+ {:else} + {#if error} +
+ {error} +
+ {/if} + {#if bluetooth} +

Please connect to a doodlebot...

+
+

...by selecting a bluetooth device

+ + {#if bleDevice} + You've selected 🤖 {bleDevice.device.name}. + {/if} +
+
+ +
-
-
-

2. Select bluetooth device

- - -
- {:else} - Uh oh! Your browser does not support bluetooth. Here's how to fix that... - TBD + {:else} + Uh oh! Your browser does not support bluetooth. Please contact an + instructor. + {/if} {/if} @@ -165,7 +163,70 @@ outline: none; } - .ip { - width: 3rem; + .open { + background-color: dodgerblue; + border: 1px solid dodgerblue; + border-radius: 4px; + box-shadow: rgba(0, 0, 0, 0.1) 0 2px 4px 0; + box-sizing: border-box; + color: #fff; + cursor: pointer; + font-family: + "Akzidenz Grotesk BQ Medium", + -apple-system, + BlinkMacSystemFont, + sans-serif; + font-size: 16px; + font-weight: 600; + outline: none; + outline: 0; + padding: 5px 15px; + text-align: center; + transform: translateY(0); + transition: + transform 150ms, + box-shadow 150ms; + user-select: none; + -webkit-user-select: none; + touch-action: manipulation; + } + + /* CSS */ + .connect { + background-color: #13aa52; + border: 1px solid #13aa52; + border-radius: 4px; + box-shadow: rgba(0, 0, 0, 0.1) 0 2px 4px 0; + box-sizing: border-box; + color: #fff; + cursor: pointer; + font-family: + "Akzidenz Grotesk BQ Medium", + -apple-system, + BlinkMacSystemFont, + sans-serif; + font-size: 16px; + font-weight: 600; + outline: none; + outline: 0; + padding: 10px 25px; + text-align: center; + transform: translateY(0); + transition: + transform 150ms, + box-shadow 150ms; + user-select: none; + -webkit-user-select: none; + touch-action: manipulation; + } + + .connect:disabled { + opacity: 0.5; + cursor: not-allowed; + } + + .connect:not(:disabled):hover { + box-shadow: rgba(0, 0, 0, 0.15) 0 3px 9px 0; + transform: translateY(-2px); } diff --git a/extensions/src/doodlebot/Doodlebot.ts b/extensions/src/doodlebot/Doodlebot.ts index f5e8f5600..efe612b8a 100644 --- a/extensions/src/doodlebot/Doodlebot.ts +++ b/extensions/src/doodlebot/Doodlebot.ts @@ -1,13 +1,14 @@ -import EventEmitter from "events"; -import { Service } from "./communication/ServiceHelper"; +/// + +import { EventEmitter } from "eventemitter3"; import UartService from "./communication/UartService"; import { followLine } from "./LineFollowing"; import { Command, DisplayKey, NetworkStatus, ReceivedCommand, SensorKey, command, display, endpoint, keyBySensor, motorCommandReceived, networkStatus, port, sensor } from "./enums"; -import { base64ToInt32Array, makeWebsocket, Max32Int, testWebSocket } from "./utils"; +import { base64ToInt32Array, deferred, makeWebsocket, Max32Int } from "./utils"; import { LineDetector } from "./LineDetection"; import { calculateArcTime } from "./TimeHelper"; +import type { BLEDeviceWithUartService } from "./ble"; -export type Services = Awaited>; export type MotorStepRequest = { /** Number of steps for the stepper (+ == forward, - == reverse) */ steps: number, @@ -19,14 +20,10 @@ export type Vector3D = { x: number, y: number, z: number }; export type Color = { red: number, green: number, blue: number, alpha: number }; export type SensorReading = number | Vector3D | Bumper | Color; export type SensorData = Doodlebot["sensorData"]; -export type NetworkCredentials = { ssid: string, password: string, ipOverride?: string }; -export type NetworkConnection = { ip: string, hostname?: string }; -export type RequestBluetooth = (callback: (bluetooth: Bluetooth) => any) => void; -export type SaveIP = (ip: string) => void; type MaybePromise = undefined | Promise; -type Pending = Record<"motor" | "wifi" | "websocket" | "video" | "image", MaybePromise> & { ip: MaybePromise }; +type Pending = Record<"motor" | "websocket" | "video" | "image", MaybePromise>; type SubscriptionTarget = Pick; @@ -40,134 +37,33 @@ type MotorCommand = "steps" | "arc" | "stop"; const trimNewtworkStatusMessage = (message: string, prefix: NetworkStatus) => message.replace(prefix, "").trim(); -const localIp = "127.0.0.1"; - const events = { stop: "motor", connect: "connect", disconnect: "disconnect", } as const; -type CreatePayload = { - credentials: NetworkCredentials, - requestBluetooth: RequestBluetooth, - saveIP: SaveIP, -} - -const msg = (content: string, type: "success" | "warning" | "error") => { - switch (type) { - case "success": - console.log(content); - break; - case "warning": - console.warn(content); - break; - case "error": - console.error(content); - break; - } -} - type BLECommunication = { onDisconnect: (...callbacks: (() => void)[]) => void, onReceive: (callback: (text: CustomEvent) => void) => void, send: (text: string) => Promise } -export default class Doodlebot { - /** - * - * @param services - * @param serviceClass - * @returns - */ - static async tryCreateService any)>( - services: BluetoothRemoteGATTService[], serviceClass: T - ): Promise> { - const found = services.find((service) => service.uuid === serviceClass.uuid); - return found ? await serviceClass.create(found) : undefined; - } - - /** - * - * @param bluetooth - * @param devicePrefix @todo unused - * @returns - */ - static async requestRobot(bluetooth: Bluetooth, ...filters: BluetoothLEScanFilter[]) { - const device = await bluetooth.requestDevice({ - filters: [ - ...(filters ?? []), - { - services: [UartService.uuid] - }, - ], - }); - - return device; - } - - /** - * Get - * @param device - * @returns - */ - static async getServices(device: BluetoothDevice) { - if (!device || !device.gatt) return null; - if (!device.gatt.connected) await device.gatt.connect(); - const services = await device.gatt.getPrimaryServices(); - const uartService = await Doodlebot.tryCreateService(services, UartService); - - return { uartService, }; - } - - static async getBLE(ble: Bluetooth, ...filters: BluetoothLEScanFilter[]) { - const robot = await Doodlebot.requestRobot(ble, ...filters); - const services = await Doodlebot.getServices(robot); - if (!services) throw new Error("Unable to connect to doodlebot's UART service"); - return { robot, services }; - } - - /** - * - * @param ble - * @param filters - * @throws - * @returns - */ - static async tryCreate( - ble: Bluetooth, - { requestBluetooth, credentials, saveIP }: CreatePayload, - ...filters: BluetoothLEScanFilter[]) { - const { robot, services } = await Doodlebot.getBLE(ble, ...filters); - return new Doodlebot({ - onReceive: (callback) => services.uartService.addEventListener("receiveText", callback), - onDisconnect: (callback) => ble.addEventListener("gattserverdisconnected", callback), - send: (text) => services.uartService.sendText(text), - }, requestBluetooth, credentials, saveIP, async (description) => { - const response = await fetch(`http://192.168.41.214:8001/webrtc`, { - method: 'POST', - body: description, - headers: { 'Content-Type': 'application/json' } - }); - console.log("INSIDE INTERNAL FUNCTION"); - const responseJson = await response.json(); - console.log("kjson", responseJson); - return responseJson; - }); - } - - private pending: Pending = { motor: undefined, wifi: undefined, websocket: undefined, ip: undefined }; +export default class Doodlebot { + private pending: Pending = { motor: undefined, websocket: undefined, video: undefined, image: undefined }; private onMotor = new EventEmitter(); private onSensor = new EventEmitter(); - private onNetwork = new EventEmitter(); private disconnectCallbacks = new Set<() => void>(); private subscriptions = new Array>(); - private connection: NetworkConnection; private websocket: WebSocket; private encoder = new TextEncoder(); + public readonly topLevelDomain = deferred(); + public readonly bleDevice = deferred(); + + private readonly ble = deferred(); + private lastDisplayedKey; private lastDisplayedType; @@ -289,39 +185,44 @@ export default class Doodlebot { private audioSocket: WebSocket; private audioCallbacks = new Set<(chunk: Float32Array) => void>(); - private pc: any; - private webrtcVideo: any; + private pc: RTCPeerConnection; + private webrtcVideo: HTMLVideoElement; public previewImage; public canvasWebrtc; - constructor( - private ble: BLECommunication, - private requestBluetooth: RequestBluetooth, - private credentials: NetworkCredentials, - private saveIP: SaveIP, - private fetchFunction - ) { - this.ble.onReceive(this.receiveTextBLE.bind(this)); - this.ble.onDisconnect(this.handleBleDisconnect.bind(this)); - this.connectionWorkflow(credentials); - + private reloadRequired?: ((msg: string) => void) | null = null; + + constructor() { + this.ble.promise.then(ble => ble.onReceive(this.receiveTextBLE.bind(this))); + this.ble.promise.then(ble => ble.onDisconnect(this.handleBleDisconnect.bind(this))); + this.bleDevice.promise.then(async ({ device, service }) => { + this.ble.resolve( + { + onReceive: (callback) => service.addEventListener("receiveText", callback), + onDisconnect: (callback) => device.addEventListener("gattserverdisconnected", callback), + send: (text) => service.sendText(text), + }); + }) + + this.connectionWorkflow(); + this.pc = new RTCPeerConnection(); this.pc.addTransceiver("video", { direction: "recvonly" }); console.log("pc", this.pc); - + this.pc.ontrack = (event) => { const stream = event.streams[0]; console.log("stream", stream); - + this.webrtcVideo = document.createElement('video'); this.webrtcVideo.srcObject = stream; this.webrtcVideo.autoplay = true; this.webrtcVideo.playsInline = true; // document.body.appendChild(this.webrtcVideo); - + this.webrtcVideo.play().catch(e => console.error("Playback error:", e)); - + this.webrtcVideo.onerror = (error) => { console.log("ERROR 2", error); }; @@ -345,39 +246,26 @@ export default class Doodlebot { }; this.webrtcVideo.requestVideoFrameCallback(handleVideoFrame); }; - - // this is an ugly hack just to get things working - // (it was crashing on my machine) - // the pc.createOffer() call below needs the IP - // but this is before the IP address has been prompted - // for. I think it works sometimes because we save the - // address for next time, but fails the first time - // it looks like this ps.createOffer call was just chucked - // here in the constructor as a quick test of webrtc video? - // -jon - const urlParams = new URLSearchParams(window.location.search); - const ip = urlParams.get("ip"); + this.pc.createOffer() .then(offer => this.pc.setLocalDescription(offer)) - //.then(() => fetch(`http://192.168.41.231:8001/webrtc`, { - .then(() => fetch(`https://${ip}/api/v1/video/webrtc`, { - method: 'POST', - body: JSON.stringify(this.pc.localDescription), - headers: { 'Content-Type': 'application/json' } - })) + .then(async () => { + const tld = await this.topLevelDomain.promise; + console.log("TLD", tld); + const webrtcResponse = await fetch(`https://${tld}/api/v1/video/webrtc`, { + method: 'POST', + body: JSON.stringify(this.pc.localDescription), + headers: { 'Content-Type': 'application/json' } + }); + console.log("webrtcResponse", webrtcResponse); + return webrtcResponse; + }) .then(response => response.json()) .then(answer => this.pc.setRemoteDescription(answer)) .catch(err => console.error("WebRTC error:", err)); - + this.previewImage = document.createElement("img"); document.body.appendChild(this.previewImage); - - // fetch(`http://192.168.41.214:8000/webrtc`, { - // method: 'POST', - // body: JSON.stringify(this.pc.localDescription), - // headers: { 'Content-Type': 'application/json' } - // }) - } private formCommand(...args: (string | number)[]) { @@ -401,11 +289,7 @@ export default class Doodlebot { private updateNetworkStatus(ipComponent: string, hostnameComponent: string) { const ip = trimNewtworkStatusMessage(ipComponent, networkStatus.ipPrefix); const hostname = trimNewtworkStatusMessage(hostnameComponent, networkStatus.hostnamePrefix); - if (ip === localIp) { - return this.onNetwork.emit(events.disconnect); - } - this.connection = { ip, hostname }; - this.onNetwork.emit(events.connect, this.connection); + this.reloadRequired?.("Network settings changed, please reload the page and reconnect to doodlebot."); } private receiveTextBLE(event: CustomEvent) { @@ -452,7 +336,7 @@ export default class Doodlebot { const [x, y, z] = parameters.map((parameter) => Number.parseFloat(parameter)); this.updateSensor(keyBySensor[command], { x, y, z }); break; - + } case sensor.light: { const [red, green, blue, alpha] = parameters.map((parameter) => Number.parseFloat(parameter)); @@ -476,19 +360,10 @@ export default class Doodlebot { const decodedMessage = decoder.decode(event.data); console.log('Received ArrayBuffer as text:', decodedMessage); } - - } - - private invalidateWifiConnection() { - this.connection = undefined; - this.pending.wifi = undefined; - this.pending.websocket = undefined; - this.websocket?.close(); - this.websocket = undefined; } private handleBleDisconnect() { - console.log("disconnected!!!"); + this.reloadRequired?.("Doodlebot bluetooth disconnected, please reload the page and reconnect to doodlebot."); for (const callback of this.disconnectCallbacks) callback(); for (const { target, event, listener } of this.subscriptions) target.removeEventListener(event, listener); } @@ -529,14 +404,14 @@ export default class Doodlebot { scheduleDisableSensor(type: SensorKey, delay = 5000) { // Reset existing timer if (this.disableTimers[type]) { - clearTimeout(this.disableTimers[type]); + clearTimeout(this.disableTimers[type]); } - + this.disableTimers[type] = setTimeout(() => { - this.disableSensor(type); - delete this.disableTimers[type]; + this.disableSensor(type); + delete this.disableTimers[type]; }, delay); - } + } /** * @@ -545,25 +420,25 @@ export default class Doodlebot { */ async getSensorReading(type: T): Promise { await this.enableSensor(type); - + const reading = this.sensorData[type]; - + // Schedule auto-disable after 5s of inactivity this.scheduleDisableSensor(type, 5000); - + return reading; } async getSingleSensorReading( type: T - ): Promise { + ): Promise { await this.enableSensor(type); - + const reading = this.sensorData[type]; - + // Schedule auto-disable after 5s of inactivity this.scheduleDisableSensor(type, 5000); - + return reading; } @@ -620,10 +495,8 @@ export default class Doodlebot { } async findImageFiles() { - while (!this.connection) { - await new Promise(resolve => setTimeout(resolve, 100)); - } - let endpoint = "https://" + this.connection.ip + "/api/v1/upload/images" + const tld = await this.topLevelDomain.promise; + let endpoint = "https://" + tld + "/api/v1/upload/images" let uploadedImages = await this.fetchAndExtractList(endpoint); return uploadedImages.filter(item => !this.imageFiles.includes(item)); } @@ -632,7 +505,7 @@ export default class Doodlebot { private streamActive: Record = {}; private stopTimers: Record = {}; - async getFacePrediction(ip: string, type: "face" | "object") { + async getFacePrediction(type: "face" | "object") { const now = Date.now(); // Clear old stop timers whenever we get a new call @@ -643,57 +516,56 @@ export default class Doodlebot { // If stream is already active → just return continuous predict if (this.streamActive[type]) { this.lastCallTime[type] = now; - this.resetStopTimer(ip, type); - return await this.getContinuousPredict(ip, type); + this.resetStopTimer(type); + return await this.getContinuousPredict(type); } // If first call or more than 1s since last call → single_predict if (!this.lastCallTime[type] || now - this.lastCallTime[type] > 1000) { this.lastCallTime[type] = now; console.log(`[${type}] Calling single_predict API...`); - return await this.callSinglePredict(ip); + return await this.callSinglePredict(); } // If called again within 1s → switch to stream console.log(`[${type}] Switching to stream API...`); this.streamActive[type] = true; this.lastCallTime[type] = now; - await this.startContinuousDetection(ip, type); - this.resetStopTimer(ip, type); - return await this.getContinuousPredict(ip, type); + await this.startContinuousDetection(type); + this.resetStopTimer(type); + return await this.getContinuousPredict(type); } - private resetStopTimer(ip: string, type: "face" | "object") { + private resetStopTimer(type: "face" | "object") { this.stopTimers[type] = setTimeout(() => { console.log(`[${type}] No calls in 5s, stopping stream...`); - this.stopContinuousDetection(ip, type); + this.stopContinuousDetection(type); }, 5000); } - async callSinglePredict(ip: string) { - const uploadEndpoint = `https://${ip}/api/v1/video/single_predict?width=320&height=240`; + async callSinglePredict() { + const tld = await this.topLevelDomain.promise; + const uploadEndpoint = `https://${tld}/api/v1/video/single_predict?width=320&height=240`; const response = await fetch(uploadEndpoint); return await response.json(); } - async startContinuousDetection(ip: string, type: "face" | "object") { - while (!this.connection) { - await new Promise(resolve => setTimeout(resolve, 100)); - } + async startContinuousDetection(type: "face" | "object") { + const tld = await this.topLevelDomain.promise; let endpoint; if (this.streamActive["face"] && this.streamActive["object"]) { - endpoint = `https://${ip}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=true&set_detect_faces=true`; + endpoint = `https://${tld}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=true&set_detect_faces=true`; } else if (this.streamActive["face"]) { - endpoint = `https://${ip}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=false&set_detect_faces=true`; + endpoint = `https://${tld}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=false&set_detect_faces=true`; } else { - endpoint = `https://${ip}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=true&set_detect_faces=false`; + endpoint = `https://${tld}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=true&set_detect_faces=false`; } console.log("starting", endpoint); await fetch(endpoint); console.log(`[${type}] Continuous detection started`); } - async stopContinuousDetection(ip: string, type: "face" | "object") { + async stopContinuousDetection(type: "face" | "object") { this.streamActive[type] = false; this.lastCallTime[type] = 0; if (this.stopTimers[type]) { @@ -701,33 +573,33 @@ export default class Doodlebot { delete this.stopTimers[type]; } let endpoint; + const tld = await this.topLevelDomain.promise; if (!this.streamActive["face"] && !this.streamActive["object"]) { - endpoint = `https://${ip}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=false&set_detect_faces=false`; - } else if (!this.streamActive["face"] ) { - endpoint = `https://${ip}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=true&set_detect_faces=false`; + endpoint = `https://${tld}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=false&set_detect_faces=false`; + } else if (!this.streamActive["face"]) { + endpoint = `https://${tld}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=true&set_detect_faces=false`; } else { - endpoint = `https://${ip}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=false&set_detect_faces=true`; + endpoint = `https://${tld}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=false&set_detect_faces=true`; } console.log("stopping", endpoint) await fetch(endpoint); console.log(`[${type}] Continuous detection stopped`); } - async getContinuousPredict(ip: string, type: "face" | "object") { - while (!this.connection) { - await new Promise(resolve => setTimeout(resolve, 100)); + async getContinuousPredict(type: "face" | "object") { + const tld = await this.topLevelDomain.promise; + + const endpoint = `https://${tld}/api/v1/video/stream_latest?screenshot=false`; + + const response = await fetch(endpoint); + if (!response.ok) { + await this.startContinuousDetection(type); + return this.getContinuousPredict(type); + } else { + return await response.json(); } - const endpoint = `https://${ip}/api/v1/video/stream_latest?screenshot=false`; - const response = await fetch(endpoint); - if (!response.ok) { - await this.startContinuousDetection(ip, type); - return this.getContinuousPredict(ip, type); - } else { - return await response.json(); - } - } getReadingLocation(axis, type, reading) { @@ -748,15 +620,15 @@ export default class Doodlebot { const firstApple = reading.objects.find(obj => obj.label === "apple"); if (firstApple) { if (axis == "x") { - return firstApple.x; + return firstApple.x; } else { - return firstApple.y; + return firstApple.y; } } else { return -1; } } else { - const firstOrange = reading.objects.find(obj => obj.label === "orange"); + const firstOrange = reading.objects.find(obj => obj.label === "orange"); if (firstOrange) { if (axis == "x") { return firstOrange.x; @@ -766,16 +638,13 @@ export default class Doodlebot { } else { return -1; } - } + } } } async findSoundFiles() { - while (!this.connection) { - await new Promise(resolve => setTimeout(resolve, 100)); - } - if (!this.connection) return []; - let endpoint = "https://" + this.connection.ip + "/api/v1/upload/sounds" + const tld = await this.topLevelDomain.promise; + let endpoint = "https://" + tld + "/api/v1/upload/sounds" let uploadedSounds = await this.fetchAndExtractList(endpoint); return uploadedSounds.filter(item => !this.soundFiles.includes(item)); } @@ -840,9 +709,9 @@ export default class Doodlebot { case "stop": if (this.isStopped) return; await this.sendBLECommand(command.motor, "s"); - // return await this.untilFinishedPending("motor", new Promise(async (resolve) => { - // this.onMotor.once(events.stop, resolve); - // })); + // return await this.untilFinishedPending("motor", new Promise(async (resolve) => { + // this.onMotor.once(events.stop, resolve); + // })); } } @@ -866,124 +735,13 @@ export default class Doodlebot { await this.sendBLECommand(command.lowPower); } - async getIPAddress() { - console.log(this.connection); - if (this.connection && this.connection.ip) { - console.log("returning") - return this.connection.ip; - } - const self = this; - const interval = setTimeout(() => this.sendBLECommand(command.network), 1000); - const ip = await new Promise(async (resolve) => { - this.onNetwork.once(events.connect, () => resolve(self.connection.ip)); - this.onNetwork.once(events.disconnect, () => resolve(null)); - }); - console.log(`Got ip: ${ip}`); - clearTimeout(interval); - return ip; - } - - async testIP(ip: string) { - if (!ip) return false; - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 2000); - try { - const resp = await fetch(`http://${ip}:8000/static/webrtc-client.html`, { signal: controller.signal }); - return resp.ok; - } - catch { - return false; - } - finally { - clearTimeout(timeout); - } - } - - setIP(ip: string) { - this.connection ??= { ip }; - this.saveIP(ip); - return this.connection.ip = ip; - } - - getStoredIPAddress() { - if (!this.connection) { return "" } - return this.connection.ip; - } - - /** - * - * @param ssid - * @param password - */ - async connectToWifi(credentials: NetworkCredentials) { - if (credentials.ipOverride) { - msg("Testing stored IP address", "warning") - const validIP = await this.testIP(credentials.ipOverride); - msg( - validIP ? "Validated stored IP address" : "Stored IP address could not be reached", - validIP ? "success" : "warning" - ) - if (validIP) return this.setIP(credentials.ipOverride); - } - - msg("Asking doodlebot for it's IP", "warning"); - - let ip = await this.getIPAddress(); - - if (ip) { - if (ip === localIp) { - msg("Doodlebot IP is local, not valid", "warning"); - } - else { - msg("Testing Doodlebot's reported IP address", "warning"); - const validIP = await this.testIP(ip); - msg( - validIP ? "Validated Doodlebot's IP address" : "Doodlebot's IP address could not be reached", - validIP ? "success" : "warning" - ) - if (validIP) return this.setIP(ip); - } - } - else { - msg("Could not retrieve IP address from doodlebot", "error") - } - - // return new Promise(async (resolve) => { - // const self = this; - // const { device } = this; - - // const reconnectToBluetooth = async () => { - // this.requestBluetooth(async (ble) => { - // msg("Reconnected to doodlebot", "success"); - // const { robot, services } = await Doodlebot.getBLE(ble); - // self.attachToBLE(robot, services); - // device.removeEventListener("gattserverdisconnected", reconnectToBluetooth); - // msg("Waiting to issue connect command", "warning"); - // await new Promise((resolve) => setTimeout(resolve, 5000)); - // msg("Testing doodlebot's IP after reconnect", "warning"); - // const ip = await self.getIPAddress(); - // msg( - // ip === localIp ? "Doodlebot's IP is local, not valid" : "Doodlebot's IP is valid", - // ip === localIp ? "warning" : "success" - // ) - // resolve(this.setIP(ip)); - // }); - // } - - // device.addEventListener("gattserverdisconnected", reconnectToBluetooth); - - // msg("Attempting to connect to wifi", "warning"); - - // await this.sendBLECommand(command.wifi, credentials.ssid, credentials.password); - // }); - } - /** * * @param credentials */ - async connectToWebsocket(ip: string) { - this.websocket = makeWebsocket(ip, '/api/v1/command'); + async connectToWebsocket() { + const tld = await this.topLevelDomain.promise; + this.websocket = makeWebsocket(tld, '/api/v1/command'); await this.untilFinishedPending("websocket", new Promise((resolve) => { const resolveAndRemove = () => { console.log("Connected to websocket"); @@ -995,60 +753,16 @@ export default class Doodlebot { })); } - async connectionWorkflow(credentials: NetworkCredentials) { - // i commented out the next line as a hack to get this working - // quickly on my machine, it probably just needs to be uncommented - // but I can't test this at the moment -jon - //await this.connectToWifi(credentials); - const urlParams = new URLSearchParams(window.location.search); // Hack for now -jon - let ip = urlParams.get("ip"); - this.setIP(ip); - - await this.connectToWebsocket(this.connection.ip); - this.detector = new LineDetector(this.connection.ip); - //await this.connectToImageWebSocket(this.connection.ip); + async connectionWorkflow() { + const tld = await this.topLevelDomain.promise; + if (this.websocket) this.websocket.close(); + await this.connectToWebsocket(); + this.detector = new LineDetector(tld); } getImageStream() { return this.canvasWebrtc; } - - - - // async getImageStream() { - // if (this.pending["websocket"]) await this.pending["websocket"]; - // if (!this.connection.ip) return; - // const image = document.createElement("img"); - // image.src = `https://${this.connection.ip}/api/v1/videopush/${endpoint.video}`; - // image.crossOrigin = "anonymous"; - // await new Promise((resolve) => image.addEventListener("load", resolve)); - // return image; - // } - - async connectToImageWebSocket(ip: string) { - // Create a WebSocket connection - this.websocket = new WebSocket(`wss://${ip}:${port.camera}`); - - // Return a promise that resolves when the WebSocket is connected - await this.untilFinishedPending("image", new Promise((resolve, reject) => { - const onOpen = () => { - console.log("Connected to WebSocket for image stream"); - this.websocket.removeEventListener("open", onOpen); - resolve(); - }; - - const onError = (err: Event) => { - console.error("WebSocket error: ", err); - reject(err); - }; - - this.websocket.addEventListener("open", onOpen); - this.websocket.addEventListener("error", onError); - - // Handle each message (which could be an image frame) - this.websocket.addEventListener("message", (event) => this.onWebSocketImageMessage(event)); - })); - } // Handle incoming image data from WebSocket onWebSocketImageMessage(event: MessageEvent) { @@ -1114,11 +828,11 @@ export default class Doodlebot { deepEqual = (a, b) => { if (a === b) return true; if (Array.isArray(a) && Array.isArray(b)) { - return a.length === b.length && a.every((val, i) => this.deepEqual(val, b[i])); + return a.length === b.length && a.every((val, i) => this.deepEqual(val, b[i])); } if (typeof a === 'object' && typeof b === 'object') { - const keysA = Object.keys(a), keysB = Object.keys(b); - return keysA.length === keysB.length && keysA.every(key => this.deepEqual(a[key], b[key])); + const keysA = Object.keys(a), keysB = Object.keys(b); + return keysA.length === keysB.length && keysA.every(key => this.deepEqual(a[key], b[key])); } return false; }; @@ -1131,9 +845,11 @@ export default class Doodlebot { let t = { aT: 0.3 }; let lastTime: number; - console.log(this.connection.ip); + const tld = await this.topLevelDomain.promise; + + console.log(tld); console.log(this.detector); - this.detector = new LineDetector(this.connection.ip); + this.detector = new LineDetector(tld); await this.detector.initialize(this); let prevLine = []; let add = 0; @@ -1153,7 +869,7 @@ export default class Doodlebot { this.lineCounter += 1; let newMotorCommands; - + if (first) { ({ motorCommands: newMotorCommands, bezierPoints: this.bezierPoints, line: this.line } = followLine( lineData, @@ -1171,13 +887,13 @@ export default class Doodlebot { lineData, prevLine, this.motorCommands, - [prevInterval/2], + [prevInterval / 2], [t.aT], add, false )); } - + lastTime = Date.now(); // Debugging statement @@ -1191,7 +907,7 @@ export default class Doodlebot { newMotorCommands[0].angle = -10; } - + if (this.iterationNumber % iterations == 0) { newMotorCommands[0].angle = this.limitArcLength(newMotorCommands[0].angle, newMotorCommands[0].radius, 2); // newMotorCommands[0].angle = this.increaseArcLength(newMotorCommands[0].angle, newMotorCommands[0].radius, ); @@ -1206,14 +922,14 @@ export default class Doodlebot { t = calculateArcTime(0, 0, newMotorCommands[0].radius, newMotorCommands[0].angle); } } - + this.motorCommands = newMotorCommands; for (const command of this.motorCommands) { let { radius, angle } = command; if ((lineData.length == 0 || !this.deepEqual(lineData, prevLine))) { if (command.distance > 0) { - this.sendWebsocketCommand("m", Math.round(12335.6*command.distance), Math.round(12335.6*command.distance), 500, 500); + this.sendWebsocketCommand("m", Math.round(12335.6 * command.distance), Math.round(12335.6 * command.distance), 500, 500); } else { this.sendBLECommand("t", radius, angle); } @@ -1221,7 +937,7 @@ export default class Doodlebot { if (this.deepEqual(lineData, prevLine) && lineData.length > 0) { console.log("LAG"); } - + } if (prevLine.length < 100 && lineData.length < 100) { add = add + 1; @@ -1229,9 +945,9 @@ export default class Doodlebot { add = 0; } } - + await new Promise((resolve) => setTimeout(resolve, waitTime)); - prevInterval = waitTime/1000; + prevInterval = waitTime / 1000; first = false; prevLine = lineData; } catch (error) { @@ -1251,7 +967,7 @@ export default class Doodlebot { } angle = Math.abs(angle); const maxAngle = (maxArcLength * 180) / ((radius + 2.93) * Math.PI); - + const returnAngle = Math.min(angle, maxAngle); // Return the limited angle return negative ? returnAngle * -1 : returnAngle; @@ -1265,17 +981,17 @@ export default class Doodlebot { } angle = Math.abs(angle); const maxAngle = (maxArcLength * 180) / ((radius + 2.93) * Math.PI); - + // Return the limited angle - return negative ? maxAngle*-1 : maxAngle; + return negative ? maxAngle * -1 : maxAngle; } - private setupAudioStream() { - if (!this.connection.ip) return false; - + private async setupAudioStream() { if (this.audioSocket) return true; - const socket = new WebSocket(`wss://${this.connection.ip}:${port.audio}`); + const tld = await this.topLevelDomain.promise; + + const socket = new WebSocket(`wss://${tld}:${port.audio}`); const self = this; socket.onopen = function (event) { @@ -1353,12 +1069,12 @@ export default class Doodlebot { console.warn("WebSocket is already sending audio data."); return; } - + this.isSendingAudio = true; let CHUNK_SIZE = 1024; - let ip = this.connection.ip; - const ws = makeWebsocket(ip, '/api/v1/speaker'); + const tld = await this.topLevelDomain.promise; + const ws = makeWebsocket(tld, '/api/v1/speaker'); ws.onopen = () => { console.log('WebSocket connection opened'); let { sampleWidth, channels, rate } = this.parseWavHeader(uint8Array); @@ -1408,10 +1124,10 @@ export default class Doodlebot { const startTime = Date.now(); const callbacks = this.audioCallbacks; - + return new Promise<{ context: AudioContext, buffer: AudioBuffer }>((resolve) => { - const accumulate = (chunk: Float32Array) => { + const accumulate = (chunk: Float32Array) => { // Check if we've exceeded our time limit if (Date.now() - startTime >= numSeconds * 1000) { callbacks.delete(accumulate); @@ -1462,30 +1178,28 @@ export default class Doodlebot { async moveEyes(direction1: string, direction2: string) { const dirMap: Record = { - center: "C", - left: "<", - right: ">", - up: "^", - down: "v", + center: "C", + left: "<", + right: ">", + up: "^", + down: "v", }; - //const websocket2 = new WebSocket(`ws://${this.connection.ip}:${8766}`); - const from = dirMap[direction1]; const to = dirMap[direction2]; - + if (!from || !to || (from != "C" && to != "C")) { - throw new Error(`Invalid direction: ${direction1}, ${direction2}`); + throw new Error(`Invalid direction: ${direction1}, ${direction2}`); } - + const movement = `${from}${to}`; - + await this.sendWebsocketCommand(command.display, movement); - } + } - async displayFile(file: string) { + async displayFile(file: string) { await this.sendWebsocketCommand(command.display, file); - } + } /** * NOTE: Consider making private @@ -1493,8 +1207,9 @@ export default class Doodlebot { * @param args * @returns */ - sendBLECommand(command: Command, ...args: (string | number)[]) { - return this.ble.send(this.formCommand(command, ...args)); + async sendBLECommand(command: Command, ...args: (string | number)[]) { + const ble = await this.ble.promise; + return ble.send(this.formCommand(command, ...args)); } /** diff --git a/extensions/src/doodlebot/LineDetection.ts b/extensions/src/doodlebot/LineDetection.ts index c03a25cdf..7175f42fe 100644 --- a/extensions/src/doodlebot/LineDetection.ts +++ b/extensions/src/doodlebot/LineDetection.ts @@ -24,7 +24,7 @@ export class LineDetector { private readonly MIN_PROCESS_INTERVAL = 100; private imageStream: HTMLImageElement | null = null; - constructor(private raspberryPiIp: string, imageStream: HTMLImageElement, width = 640, height = 480) { + constructor(private raspberryPiIp: string, imageStream: HTMLImageElement = null, width = 640, height = 480) { debug.info('Initializing LineDetector', { raspberryPiIp, width, height }); this.width = width; @@ -110,56 +110,56 @@ export class LineDetector { try { - try { + try { - debug.info('Clearing canvas and drawing new image'); - this.ctx.clearRect(0, 0, this.width, this.height); - this.ctx.drawImage(this.imageStream!, 0, 0, this.width, this.height); + debug.info('Clearing canvas and drawing new image'); + this.ctx.clearRect(0, 0, this.width, this.height); + this.ctx.drawImage(this.imageStream!, 0, 0, this.width, this.height); - debug.info('Getting image data from canvas'); - const imageData = this.ctx.getImageData(0, 0, this.width, this.height); + debug.info('Getting image data from canvas'); + const imageData = this.ctx.getImageData(0, 0, this.width, this.height); - debug.info('Processing image data'); - const lineCoordinates = this.processImageData(imageData); + debug.info('Processing image data'); + const lineCoordinates = this.processImageData(imageData); - if (lineCoordinates.length > 0) { - debug.info('Line detected successfully', { - points: lineCoordinates.length, - firstPoint: lineCoordinates[0], - lastPoint: lineCoordinates[lineCoordinates.length - 1] - }); + if (lineCoordinates.length > 0) { + debug.info('Line detected successfully', { + points: lineCoordinates.length, + firstPoint: lineCoordinates[0], + lastPoint: lineCoordinates[lineCoordinates.length - 1] + }); - this.lastDetectedLine = lineCoordinates; + this.lastDetectedLine = lineCoordinates; - if (this.collectLine) { - this.allCoordinates.push(lineCoordinates); - this.frameCount++; - this.collectLine = false; - debug.info('Line collected, frame count:', this.frameCount); - } - - this.lastProcessTime = now; - this.isProcessing = false; - debug.timeEnd('detectLine'); - return lineCoordinates; - } else { - this.lastDetectedLine = lineCoordinates; + if (this.collectLine) { + this.allCoordinates.push(lineCoordinates); + this.frameCount++; + this.collectLine = false; + debug.info('Line collected, frame count:', this.frameCount); } - debug.warn('No line detected in this attempt'); - attempt++; - const backoffTime = 100 * Math.pow(2, attempt); - debug.info(`Waiting ${backoffTime}ms before next attempt`); + this.lastProcessTime = now; + this.isProcessing = false; + debug.timeEnd('detectLine'); + return lineCoordinates; + } else { + this.lastDetectedLine = lineCoordinates; + } - } catch (error) { - debug.error(`Processing attempt ${attempt + 1} failed:`, error); - attempt++; + debug.warn('No line detected in this attempt'); + attempt++; + const backoffTime = 100 * Math.pow(2, attempt); + debug.info(`Waiting ${backoffTime}ms before next attempt`); - + } catch (error) { + debug.error(`Processing attempt ${attempt + 1} failed:`, error); + attempt++; - const backoffTime = 100 * Math.pow(2, attempt); - debug.info(`Waiting ${backoffTime}ms before retry`); - } + + + const backoffTime = 100 * Math.pow(2, attempt); + debug.info(`Waiting ${backoffTime}ms before retry`); + } } catch (error) { debug.error('Error getting image stream:', error); diff --git a/extensions/src/doodlebot/ble.ts b/extensions/src/doodlebot/ble.ts new file mode 100644 index 000000000..198fe67a3 --- /dev/null +++ b/extensions/src/doodlebot/ble.ts @@ -0,0 +1,24 @@ +import UartService from "./communication/UartService"; + +export type BLEDeviceWithUartService = { device: BluetoothDevice, service: UartService }; + +export const getBLEDeviceWithUartService = async ( + bluetooth: Bluetooth, ...filters: BluetoothLEScanFilter[] +): Promise => { + const device = await bluetooth.requestDevice({ + filters: [ + ...(filters ?? []), + { + services: [UartService.uuid] + }, + ], + }); + if (!device) return { error: "No device selected" }; + if (!device.gatt) return { error: "No GATT server found" }; + if (!device.gatt.connected) await device.gatt.connect(); + const services = await device.gatt.getPrimaryServices(); + const found = services.find((service) => service.uuid === UartService.uuid); + if (!found) return { error: "UART service not found" }; + const service = await UartService.create(found); + return { device, service }; +} diff --git a/extensions/src/doodlebot/index.ts b/extensions/src/doodlebot/index.ts index 67035e8f9..35aee604e 100644 --- a/extensions/src/doodlebot/index.ts +++ b/extensions/src/doodlebot/index.ts @@ -1,19 +1,14 @@ import { Environment, ExtensionMenuDisplayDetails, extension, block, buttonBlock, BlockUtilityWithID, scratch } from "$common"; import { DisplayKey, displayKeys, command, type Command, SensorKey, sensorKeys, units, keyBySensor, sensor } from "./enums"; -import Doodlebot, { NetworkCredentials } from "./Doodlebot"; -import FileArgument from './FileArgument.svelte'; -import { splitArgsString } from "./utils"; +import Doodlebot from "./Doodlebot"; import EventEmitter from "events"; -import { categoryByGesture, classes, emojiByGesture, gestureDetection, gestureMenuItems, gestures, objectDetection } from "./detection"; +import { categoryByGesture } from "./detection"; //import { createLineDetector } from "./LineDetection"; -import { line0, line1, line2, line3, line4, line5, line6, line7, line8 } from './Points'; -import { followLine } from "./LineFollowing"; -import { createLineDetector } from "./LineDetection"; import tmPose from '@teachablemachine/pose'; -import { calculateArcTime } from "./TimeHelper"; import tmImage from '@teachablemachine/image'; import * as speechCommands from '@tensorflow-models/speech-commands'; import JSZip from 'jszip'; +import type { BLEDeviceWithUartService } from "./ble"; const details: ExtensionMenuDisplayDetails = { name: "Doodlebot", @@ -49,7 +44,9 @@ export var imageFiles: string[] = []; export var soundFiles: string[] = []; export default class DoodlebotBlocks extends extension(details, "ui", "customArguments", "indicators", "video", "drawable") { - doodlebot: Doodlebot; + doodlebot = new Doodlebot(); + connected = false; + private indicator: Promise<{ close(): void; }>; private lineDetector: (() => Promise) | null = null; bluetoothEmitter = new EventEmitter(); @@ -88,8 +85,6 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg soundDictionary: {} | { string: string[] }; costumeDictionary: {} | { string: string[] }; - externalIp: string - voice_id: number; pitch_value: number; @@ -122,13 +117,12 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg this.pitch_value = 0; this.soundDictionary = {}; this.costumeDictionary = {}; - this.setIndicator("disconnected"); - if (window.isSecureContext) this.openUI("Connect") - else this.connectToDoodlebotWithExternalBLE(); + //requestAnimationFrame(() => this.setIndicator("disconnected")); + this.openUI("Connect") this._loop(); env.runtime.on("TARGETS_UPDATE", async () => { await this.setDictionaries(); - }) + }) await this.setDictionaries(); @@ -140,7 +134,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg // const response = await fetch(url, { // method: "POST" // }); - + // if (!response.ok) { // const text = await response.text(); // console.error("Error setting voice/pitch:", text); @@ -150,7 +144,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg // } // env.runtime.on("PROJECT_RUN_START", async () => { - + // }) // env.runtime.on("PROJECT_RUN_STOP", async () => { // if (this.blocksRun > 10) { @@ -176,7 +170,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg // } // }) - + } async setDictionaries() { @@ -251,173 +245,23 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg }); } - private async connectToDoodlebotWithExternalBLE() { - // A few globals that currently must be set the same across the playground and the https frontend - const handshakeMessage = "doodlebot"; - const disconnectMessage = "disconnected"; - const commandCompleteIdentifier = "done"; - - const urlParams = new URLSearchParams(window.location.search); // Hack for now -jon - - const ip = urlParams.get("ip"); - - if (!ip) { - alert("No IP address provided. Please provide an IP address in the URL query string."); - return; - } - - this.externalIp = ip; - - const networkCredentials: NetworkCredentials = { - ssid: "dummy", // NOTE: When using the external BLE, it is assumed a valid ip address will be provided, and thus there is no need for wifi credentials - password: "dummy", // NOTE: When using the external BLE, it is assumed a valid ip address will be provided, and thus there is no need for wifi credentials - ipOverride: ip - } - - type ExternalPageDetails = { source: MessageEventSource, targetOrigin: string } - - const { source, targetOrigin } = await new Promise((resolve) => { - const onInitialMessage = ({ data, source, origin }: MessageEvent) => { - if (typeof data !== "string" || data !== handshakeMessage) return; - window.removeEventListener("message", onInitialMessage); - console.log("posting ready"); - source.postMessage("ready", { targetOrigin: origin }) - resolve({ source, targetOrigin: origin }); - } - window.addEventListener("message", onInitialMessage); - }); - - console.log("source", source); - console.log("target origin", targetOrigin); - - const doodlebot = new Doodlebot( - { - send: (text) => new Promise(resolve => { - const onMessageReturn = ({ data, origin }: MessageEvent) => { - if (origin !== targetOrigin || !data.includes(text) || !data.includes(commandCompleteIdentifier)) { - console.log("error -- source"); - return; - } - window.removeEventListener("message", onMessageReturn); - resolve(); - } - window.addEventListener("message", onMessageReturn); - console.log("posting message"); - source.postMessage(text, { targetOrigin }); - }), - - onReceive: (callback) => { - window.addEventListener('message', ({ data, origin }) => { - console.log("RECEIVED", data); - if (origin !== targetOrigin || data === disconnectMessage || data.includes(commandCompleteIdentifier)) { - console.log("error 2 -- source", data); - return; - } - callback(new CustomEvent("ble", { detail: data })); - }); - }, - - onDisconnect: () => { - window.addEventListener("message", ({ data, origin }) => { - if (origin !== targetOrigin || data !== disconnectMessage) return; - this.setIndicator("disconnected"); - alert("Disconnected from robot"); // Decide how to handle (maybe direct user to close window and go back to https) - }); - }, - }, - () => alert("requestBluetooth called"), // placeholder - networkCredentials, - () => alert("save IP called"), // placeholder, - async (description) => { - // Send the fetch request to the source - console.log("INSIDE FETCH 2"); - - return new Promise((resolve, reject) => { - console.log("INSIDE PROMISE 2"); - const fetchReturn = (event: MessageEvent) => { - console.log("inside return"); - if (event.origin !== targetOrigin) { - console.log("ERROR", event.origin, targetOrigin); - return; - } - if (!event.data.startsWith("fetchResponse---")) { - console.log("ERROR", event.data); - return; - } - const urlReturned = event.data.split("---")[1]; - if ("webrtc" != urlReturned) { - console.log("URL NOT SAME"); - return; - } - const response = event.data.split("---")[2]; - console.log("RESPONSE", JSON.parse(response)); - window.removeEventListener('message', fetchReturn); - resolve(JSON.parse(response)); - } - console.log("adding return"); - window.addEventListener('message', fetchReturn); - console.log("posting message", `fetch---webrtc---${description}`, targetOrigin); - source.postMessage(`fetch---webrtc---${description}`, { targetOrigin }); - }); - } - ) - doodlebot.fetch = async (url: string, type: string, options?: string) => { - // Send the fetch request to the source - return new Promise((resolve, reject) => { - const fetchReturn = (event: MessageEvent) => { - if (event.origin !== targetOrigin) { - console.log("ERROR", event.origin, targetOrigin); - return; - } - if (!event.data.startsWith("fetchResponse---")) { - console.log("ERROR", event.data); - return; - } - const urlReturned = event.data.split("---")[1]; - if (url != urlReturned) { - console.log("URL NOT SAME"); - return; - } - const response = event.data.split("---")[2]; - console.log("RESPONSE", response); - window.removeEventListener('message', fetchReturn); - resolve(response); - } - console.log("adding return"); - window.addEventListener('message', fetchReturn); - console.log("posting message", `fetch---${type}--${url}`, targetOrigin); - if (options) { - source.postMessage(`fetch---${type}---${url}---${JSON.stringify(options)}`, { targetOrigin }); - } else { - source.postMessage(`fetch---${type}---${url}`, { targetOrigin }); - } - }); - } - doodlebot.setIP(ip); - this.setDoodlebot(doodlebot); - - - } - getCurrentSounds(id): string[] { return (this.soundDictionary && this.soundDictionary[id]) ? Object.keys(this.soundDictionary[id]) : []; } - async setDoodlebot(doodlebot: Doodlebot) { - this.doodlebot = doodlebot; - await this.setIndicator("connected"); + async setDoodlebot(topLevelDomain: string, bluetooth: BLEDeviceWithUartService) { + this.doodlebot.topLevelDomain.resolve(topLevelDomain); + this.doodlebot.bleDevice.resolve(bluetooth); - const urlParams = new URLSearchParams(window.location.search); // Hack for now -jon - const ip = urlParams.get("ip"); - this.doodlebot.setIP(ip); + await this.setIndicator("connected"); try { - imageFiles = await doodlebot.findImageFiles(); - soundFiles = await doodlebot.findSoundFiles(); + imageFiles = await this.doodlebot.findImageFiles(); + soundFiles = await this.doodlebot.findSoundFiles(); } catch (e) { //this.openUI("ArrayError"); } - + // Wait a short moment to ensure connection is established await new Promise(resolve => setTimeout(resolve, 1000)); @@ -433,7 +277,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg } }); - + try { if (this.SOCIAL && Math.random() < this.socialness && this.doodlebot) { await this.doodlebot.display("happy"); @@ -445,19 +289,6 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg } await this.doodlebot.display("happy"); - - try { - console.log("FETCHING"); - const ip = await this.getIPAddress(); - console.log(doodlebot.fetch); - let tempImage = await doodlebot.fetch(`http://${ip}:8080/images`, "text"); - imageFiles = doodlebot.extractList(tempImage); - console.log("FILES", imageFiles) - let tempSound = await doodlebot.fetch(`http://${ip}:8080/sounds`, "text"); - soundFiles = doodlebot.extractList(tempSound); - } catch (e) { - //this.openUI("ArrayError"); - } } async setIndicator(status: "connected" | "disconnected") { @@ -501,7 +332,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg //console.error("Failed to get image stream"); return; } - + let stageWidth = 480; let stageHeight = 360; @@ -509,22 +340,22 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg resizedCanvas.width = stageWidth; resizedCanvas.height = stageHeight; const resizedCtx = resizedCanvas.getContext('2d'); - + const drawable = this.createDrawable(resizedCanvas); // draw from resized version drawable.setVisible(true); - + const update = () => { const latest = this.doodlebot?.getImageStream(); if (!latest) return; - + // Draw the current stream into the resized canvas resizedCtx.clearRect(0, 0, stageWidth, stageHeight); resizedCtx.drawImage(latest, 0, 0, stageWidth, stageHeight); drawable.update(resizedCanvas); - + requestAnimationFrame(update); }; - + requestAnimationFrame(update); return drawable; } @@ -545,7 +376,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg // async setSocialness(value: number) { // // Ensure value is between 0 and 1 // this.socialness = Math.max(0, Math.min(1, value)); - + // if (this.SOCIAL && Math.random() < this.socialness) { // await this.doodlebot?.display("happy"); // await this.speakText(`I'll be ${Math.round(this.socialness * 100)}% social from now on!`); @@ -574,15 +405,15 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg type: "command", text: (voice, pitch) => `set voice to ${voice} and pitch to ${pitch}`, args: [ - { type: "number", defaultValue: 1, name: "voice" }, - { type: "number", defaultValue: 0, name: "pitch" } + { type: "number", defaultValue: 1 }, + { type: "number", defaultValue: 0 } ] }) async setVoiceAndPitch(voice: number, pitch: number) { this.voice_id = voice; this.pitch_value = pitch; } - + // @block({ // type: "command", @@ -592,14 +423,14 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg // async repeatAfterMe(seconds: number) { // // Record the audio // const { context, buffer } = await this.doodlebot?.recordAudio(seconds); - + // // Convert to WAV format // const wavBlob = await this.saveAudioBufferToWav(buffer); // const arrayBuffer = await wavBlob.arrayBuffer(); - + // // Send the audio data directly to the Doodlebot for playback // await this.doodlebot.sendAudioData(new Uint8Array(arrayBuffer)); - + // // Wait until playback is complete (approximately buffer duration) // const playbackDuration = buffer.duration * 1000; // convert to milliseconds // await new Promise(resolve => setTimeout(resolve, playbackDuration)); @@ -764,7 +595,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg type: "reporter", text: (axis: string, sensor: SensorKey) => `get ${axis} of ${sensor} sensor`, args: [ - {type: 'string', options: ["x", "y", "z"], defaultValue: 'x'}, + { type: 'string', options: ["x", "y", "z"], defaultValue: 'x' }, { type: "string", options: ["gyroscope", "accelerometer"], defaultValue: "gyroscope" } ] }) @@ -776,7 +607,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg return `${JSON.stringify(reading[axis])} ${units[sensor]}`; } - + @(scratch.hat` when ${{ type: "string", options: ["battery", "temperature", "humidity", "pressure", "distance", "altimeter"], defaultValue: "battery" }} @@ -820,7 +651,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg }) whenBumperPressed(bumber: typeof bumperOptions[number], condition: "release" | "pressed") { const isPressed = this.doodlebot?.getSensorReadingSync("bumper"); - + const isPressedCondition = condition === "pressed"; if (!isPressed) { return false; @@ -931,7 +762,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg (self.costumeDictionary && self.costumeDictionary[self.runtime._editingTarget.id]) ? Object.keys(self.costumeDictionary[self.runtime._editingTarget.id]) : [] as any[] ).filter((item: string) => item != "costume9999.png") }, defaultValue: "happy" - }, {type: "string", defaultValue: 1}] + }, { type: "string", defaultValue: 1 }] })) async setDisplayForSeconds(display: DisplayKey | string, seconds: number) { const lastDisplayedKey = this.doodlebot.getLastDisplayedKey(); @@ -941,28 +772,20 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg await this.uploadFile("image", this.costumeDictionary[this.runtime._editingTarget.id][display]); await this.setArrays(); await this.doodlebot.displayFile("costume9999.png"); - await new Promise(resolve => setTimeout(resolve, seconds*1000)); + await new Promise(resolve => setTimeout(resolve, seconds * 1000)); } else if (imageFiles.includes(display)) { await this.doodlebot?.displayFile(display); - await new Promise(resolve => setTimeout(resolve, seconds*1000)); + await new Promise(resolve => setTimeout(resolve, seconds * 1000)); } else { await this.doodlebot?.display(display as DisplayKey); - await new Promise(resolve => setTimeout(resolve, seconds*1000)); + await new Promise(resolve => setTimeout(resolve, seconds * 1000)); } if (lastDisplayedType == "text") { await this.doodlebot.displayText(lastDisplayedKey); } else { await this.doodlebot.display(lastDisplayedKey); } - - } - async getIPAddress() { - if (window.isSecureContext) { - return this.doodlebot?.getStoredIPAddress(); - } else { - return this.externalIp; - } } // @block({ @@ -1077,7 +900,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg // await new Promise((resolve) => setTimeout(resolve, audioDuration * 1000)); // } - + async setArrays() { imageFiles = await this.doodlebot.findImageFiles(); soundFiles = await this.doodlebot.findSoundFiles(); @@ -1086,13 +909,13 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg async uploadFile(type: string, blobURL: string) { console.log("BEFORE IP"); - const ip = await this.getIPAddress(); + const tld = await this.doodlebot.topLevelDomain.promise; console.log("GOT IP"); let uploadEndpoint; if (type == "sound") { - uploadEndpoint = "https://" + ip + "/api/v1/upload/sounds_upload"; + uploadEndpoint = "https://" + tld + "/api/v1/upload/sounds_upload"; } else { - uploadEndpoint = "https://" + ip + "/api/v1/upload/img_upload"; + uploadEndpoint = "https://" + tld + "/api/v1/upload/img_upload"; } try { @@ -1106,50 +929,38 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg throw new Error(`Failed to fetch Blob from URL: ${blobURL}`); } const blob = await response1.blob(); - if (window.isSecureContext) { - console.log("BEFORE BLOB 2"); - - console.log("AFTER BLOB 2"); - // Convert Blob to File - const file = new File([blob], components[0], { type: blob.type }); - const formData = new FormData(); - formData.append("file", file); - - console.log("file"); - console.log(file); - console.log("BEFORE FETCH"); - const response2 = await fetch(uploadEndpoint, { - method: "POST", - body: formData, - }); - console.log("AFTER FETCH"); - console.log(response2); - - if (!response2.ok) { - throw new Error(`Failed to upload file: ${response2.statusText}`); - } + console.log("BEFORE BLOB 2"); + + console.log("AFTER BLOB 2"); + // Convert Blob to File + const file = new File([blob], components[0], { type: blob.type }); + const formData = new FormData(); + formData.append("file", file); + + console.log("file"); + console.log(file); + console.log("BEFORE FETCH"); + const response2 = await fetch(uploadEndpoint, { + method: "POST", + body: formData, + }); + console.log("AFTER FETCH"); + console.log(response2); - console.log("File uploaded successfully"); - this.setArrays(); - } else { - const base64 = await this.blobToBase64(blob); - const payload = { - filename: components[0], - content: base64, - mimeType: blob.type, - }; - const response2 = await this.doodlebot.fetch(uploadEndpoint, "file_upload", payload); + if (!response2.ok) { + throw new Error(`Failed to upload file: ${response2.statusText}`); } - + + console.log("File uploaded successfully"); + this.setArrays(); } catch (error) { console.error("Error:", error); } } async callSinglePredict() { - const ip = await this.getIPAddress(); - return await this.doodlebot.callSinglePredict(this.getIPAddress()); - + return await this.doodlebot.callSinglePredict(); + } @block({ @@ -1161,19 +972,17 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg ] }) async getSinglePredict2s(location: string, type: "face" | "object") { - const ip = await this.getIPAddress(); - const reading = await this.doodlebot.getFacePrediction(ip, type); + const reading = await this.doodlebot.getFacePrediction(type); return this.doodlebot.getReadingLocation(location, type == "object" ? "apple" : type, reading); } @block({ type: "Boolean", text: (type) => `is ${type} detected`, - args: [{ type: "string", options: ["face", "apple", "orange"], defaultValue: "face" }] + arg: { type: "string", options: ["face", "apple", "orange"], defaultValue: "face" } }) async isFaceDetected(type: string) { - const ip = await this.getIPAddress(); - const reading = await this.doodlebot.getFacePrediction(ip, "face"); + const reading = await this.doodlebot.getFacePrediction("face"); const x = this.doodlebot.getReadingLocation("x", type, reading); const y = this.doodlebot.getReadingLocation("y", type, reading); if (x == -1 && y == -1) { @@ -1212,17 +1021,10 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg await this.doodlebot?.setVolume(volume) } - - - // @block({ - // type: "reporter", - // text: "get IP address" - // }) - async getIP() { - return this.doodlebot?.getIPAddress(); - } + + // @block({ // type: "command", @@ -1314,67 +1116,67 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg writeString(view: DataView, offset: number, text: string) { for (let i = 0; i < text.length; i++) { - view.setUint8(offset + i, text.charCodeAt(i)); + view.setUint8(offset + i, text.charCodeAt(i)); } } async saveAudioBufferToWav(buffer) { function createWavHeader(buffer) { - const numChannels = buffer.numberOfChannels; - const sampleRate = buffer.sampleRate / 4; - const bitsPerSample = 16; // 16-bit PCM - const blockAlign = (numChannels * bitsPerSample) / 8; - const byteRate = sampleRate * blockAlign; - const dataLength = buffer.length * numChannels * 2; // 16-bit PCM = 2 bytes per sample - const header = new ArrayBuffer(44); - const view = new DataView(header); - // "RIFF" chunk descriptor - writeString(view, 0, "RIFF"); - view.setUint32(4, 36 + dataLength, true); // File size - 8 bytes - writeString(view, 8, "WAVE"); - // "fmt " sub-chunk - writeString(view, 12, "fmt "); - view.setUint32(16, 16, true); // Sub-chunk size (16 for PCM) - view.setUint16(20, 1, true); // Audio format (1 = PCM) - view.setUint16(22, numChannels, true); // Number of channels - view.setUint32(24, sampleRate, true); // Sample rate - view.setUint32(28, byteRate, true); // Byte rate - view.setUint16(32, blockAlign, true); // Block align - view.setUint16(34, bitsPerSample, true); // Bits per sample - // "data" sub-chunk - writeString(view, 36, "data"); - view.setUint32(40, dataLength, true); // Data length - console.log("WAV Header:", new Uint8Array(header)); - return header; + const numChannels = buffer.numberOfChannels; + const sampleRate = buffer.sampleRate / 4; + const bitsPerSample = 16; // 16-bit PCM + const blockAlign = (numChannels * bitsPerSample) / 8; + const byteRate = sampleRate * blockAlign; + const dataLength = buffer.length * numChannels * 2; // 16-bit PCM = 2 bytes per sample + const header = new ArrayBuffer(44); + const view = new DataView(header); + // "RIFF" chunk descriptor + writeString(view, 0, "RIFF"); + view.setUint32(4, 36 + dataLength, true); // File size - 8 bytes + writeString(view, 8, "WAVE"); + // "fmt " sub-chunk + writeString(view, 12, "fmt "); + view.setUint32(16, 16, true); // Sub-chunk size (16 for PCM) + view.setUint16(20, 1, true); // Audio format (1 = PCM) + view.setUint16(22, numChannels, true); // Number of channels + view.setUint32(24, sampleRate, true); // Sample rate + view.setUint32(28, byteRate, true); // Byte rate + view.setUint16(32, blockAlign, true); // Block align + view.setUint16(34, bitsPerSample, true); // Bits per sample + // "data" sub-chunk + writeString(view, 36, "data"); + view.setUint32(40, dataLength, true); // Data length + console.log("WAV Header:", new Uint8Array(header)); + return header; } function writeString(view, offset, string) { - for (let i = 0; i < string.length; i++) { - view.setUint8(offset + i, string.charCodeAt(i)); - } + for (let i = 0; i < string.length; i++) { + view.setUint8(offset + i, string.charCodeAt(i)); + } } function interleave(buffer) { - const numChannels = buffer.numberOfChannels; - const length = buffer.length * numChannels; - const result = new Float32Array(length); - const channelData = []; - for (let i = 0; i < numChannels; i++) { - channelData.push(buffer.getChannelData(i)); - } - let index = 0; - for (let i = 0; i < buffer.length; i++) { - for (let j = 0; j < numChannels; j++) { - result[index++] = channelData[j][i]; - } + const numChannels = buffer.numberOfChannels; + const length = buffer.length * numChannels; + const result = new Float32Array(length); + const channelData = []; + for (let i = 0; i < numChannels; i++) { + channelData.push(buffer.getChannelData(i)); + } + let index = 0; + for (let i = 0; i < buffer.length; i++) { + for (let j = 0; j < numChannels; j++) { + result[index++] = channelData[j][i]; } - console.log("Interleaved data:", result); - return result; + } + console.log("Interleaved data:", result); + return result; } function floatTo16BitPCM(output, offset, input) { - for (let i = 0; i < input.length; i++, offset += 2) { - let s = Math.max(-1, Math.min(1, input[i])); // Clamp to [-1, 1] - s = s < 0 ? s * 0x8000 : s * 0x7FFF; // Convert to 16-bit PCM - output.setInt16(offset, s, true); // Little-endian - } + for (let i = 0; i < input.length; i++, offset += 2) { + let s = Math.max(-1, Math.min(1, input[i])); // Clamp to [-1, 1] + s = s < 0 ? s * 0x8000 : s * 0x7FFF; // Convert to 16-bit PCM + output.setInt16(offset, s, true); // Little-endian + } } const header = createWavHeader(buffer); const interleaved = interleave(buffer); @@ -1389,68 +1191,68 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg console.log("Expected data length:", header.byteLength + interleaved.length * 2); // Return a Blob return new Blob([wavBuffer], { type: "audio/wav" }); -} + } -generateWAV(interleaved: Float32Array, sampleRate: number): Uint8Array { - const numChannels = 1; // Mono - const bitsPerSample = 16; - const byteRate = sampleRate * numChannels * (bitsPerSample / 8); - const blockAlign = numChannels * (bitsPerSample / 8); - const dataLength = interleaved.length * (bitsPerSample / 8); - const bufferLength = 44 + dataLength; - const buffer = new ArrayBuffer(bufferLength); - const view = new DataView(buffer); - // RIFF header - this.writeString(view, 0, "RIFF"); - view.setUint32(4, bufferLength - 8, true); // File size - this.writeString(view, 8, "WAVE"); - // fmt subchunk - this.writeString(view, 12, "fmt "); - view.setUint32(16, 16, true); // Subchunk size - view.setUint16(20, 1, true); // PCM format - view.setUint16(22, numChannels, true); // Channels - view.setUint32(24, sampleRate, true); // Sample rate - view.setUint32(28, byteRate, true); // Byte rate - view.setUint16(32, blockAlign, true); // Block align - view.setUint16(34, bitsPerSample, true); // Bits per sample - // data subchunk - this.writeString(view, 36, "data"); - view.setUint32(40, dataLength, true); - // PCM data - const offset = 44; - for (let i = 0; i < interleaved.length; i++) { + generateWAV(interleaved: Float32Array, sampleRate: number): Uint8Array { + const numChannels = 1; // Mono + const bitsPerSample = 16; + const byteRate = sampleRate * numChannels * (bitsPerSample / 8); + const blockAlign = numChannels * (bitsPerSample / 8); + const dataLength = interleaved.length * (bitsPerSample / 8); + const bufferLength = 44 + dataLength; + const buffer = new ArrayBuffer(bufferLength); + const view = new DataView(buffer); + // RIFF header + this.writeString(view, 0, "RIFF"); + view.setUint32(4, bufferLength - 8, true); // File size + this.writeString(view, 8, "WAVE"); + // fmt subchunk + this.writeString(view, 12, "fmt "); + view.setUint32(16, 16, true); // Subchunk size + view.setUint16(20, 1, true); // PCM format + view.setUint16(22, numChannels, true); // Channels + view.setUint32(24, sampleRate, true); // Sample rate + view.setUint32(28, byteRate, true); // Byte rate + view.setUint16(32, blockAlign, true); // Block align + view.setUint16(34, bitsPerSample, true); // Bits per sample + // data subchunk + this.writeString(view, 36, "data"); + view.setUint32(40, dataLength, true); + // PCM data + const offset = 44; + for (let i = 0; i < interleaved.length; i++) { const sample = Math.max(-1, Math.min(1, interleaved[i])); view.setInt16(offset + i * 2, sample < 0 ? sample * 0x8000 : sample * 0x7FFF, true); + } + return new Uint8Array(buffer); + } + + createAndSaveWAV(interleaved, sampleRate) { + // Step 1: Get interleaved audio data and sample rate + // Step 2: Generate WAV file + const wavData = this.generateWAV(interleaved, sampleRate); + // Step 3: Save or process the WAV file + // Example: Create a Blob and download the file + const blob = new Blob([wavData], { type: "audio/wav" }); + const url = URL.createObjectURL(blob); + // Create a link to download the file + const a = document.createElement("a"); + a.href = url; + a.download = "output.wav"; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + return blob; + } + + blobToBase64(blob) { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onloadend = () => resolve(reader.result); // "data:;base64," + reader.onerror = reject; + reader.readAsDataURL(blob); + }); } - return new Uint8Array(buffer); -} - -createAndSaveWAV(interleaved, sampleRate) { - // Step 1: Get interleaved audio data and sample rate - // Step 2: Generate WAV file - const wavData = this.generateWAV(interleaved, sampleRate); - // Step 3: Save or process the WAV file - // Example: Create a Blob and download the file - const blob = new Blob([wavData], { type: "audio/wav" }); - const url = URL.createObjectURL(blob); - // Create a link to download the file - const a = document.createElement("a"); - a.href = url; - a.download = "output.wav"; - document.body.appendChild(a); - a.click(); - document.body.removeChild(a); - return blob; -} - -blobToBase64(blob) { - return new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onloadend = () => resolve(reader.result); // "data:;base64," - reader.onerror = reject; - reader.readAsDataURL(blob); - }); -} async sendAudioFileToChatEndpoint(file, endpoint, blob, seconds) { console.log("sending audio file"); @@ -1462,111 +1264,111 @@ blobToBase64(blob) { //audio.play(); try { - let response; - let uint8array; - // if (window.isSecureContext) { - - // if (endpoint == "repeat_after_me") { - // const eventSource = new EventSource("http://doodlebot.media.mit.edu/viseme-events"); - - // eventSource.onmessage = (event) => { - // console.log("Received viseme event:", event.data); - // try { - // const data = JSON.parse(event.data); - // const visemeId = data.visemeId; - // const offsetMs = data.offsetMs; - - // // You can customize which viseme IDs should trigger a command. - // // For now, all non-silence visemes trigger it. - // if (visemeId !== 0) { - // setTimeout(() => { - // this.doodlebot.display("happy"); - // console.log("DISPLAYING"); - // }, offsetMs); - // } - // } catch (err) { - // console.error("Failed to parse viseme event:", err); - // } - // }; - - // eventSource.onerror = (err) => { - // console.error("EventSource failed:", err); - // eventSource.close(); - // }; - // } - - response = await fetch(url, { - method: "POST", - body: formData, - }); + let response; + let uint8array; + // if (window.isSecureContext) { + + // if (endpoint == "repeat_after_me") { + // const eventSource = new EventSource("http://doodlebot.media.mit.edu/viseme-events"); + + // eventSource.onmessage = (event) => { + // console.log("Received viseme event:", event.data); + // try { + // const data = JSON.parse(event.data); + // const visemeId = data.visemeId; + // const offsetMs = data.offsetMs; + + // // You can customize which viseme IDs should trigger a command. + // // For now, all non-silence visemes trigger it. + // if (visemeId !== 0) { + // setTimeout(() => { + // this.doodlebot.display("happy"); + // console.log("DISPLAYING"); + // }, offsetMs); + // } + // } catch (err) { + // console.error("Failed to parse viseme event:", err); + // } + // }; + + // eventSource.onerror = (err) => { + // console.error("EventSource failed:", err); + // eventSource.close(); + // }; + // } + + response = await fetch(url, { + method: "POST", + body: formData, + }); - if (!response.ok) { - const errorText = await response.text(); - console.log("Error response:", errorText); - throw new Error(`HTTP error! status: ${response.status}`); - } + if (!response.ok) { + const errorText = await response.text(); + console.log("Error response:", errorText); + throw new Error(`HTTP error! status: ${response.status}`); + } + + const textResponse = response.headers.get("text-response"); + console.log("Text Response:", textResponse); + + const blob = await response.blob(); + const audioUrl = URL.createObjectURL(blob); + console.log("Audio URL:", audioUrl); + + const audio = new Audio(audioUrl); + const array = await blob.arrayBuffer(); + uint8array = new Uint8Array(array); + // } else { + // const base64 = await this.blobToBase64(blob); + // const payload = { + // filename: file.name, + // content: base64, + // mimeType: blob.type, + // }; + // response = await this.doodlebot.fetch(endpoint, "chatgpt", payload); + // uint8array = new Uint8Array([...atob(response)].map(char => char.charCodeAt(0))); + // } + const interval = 50; // 0.2 seconds in milliseconds + const endTime = Date.now() + 1 * 1000; + + this.doodlebot.sendAudioData(uint8array); + while (Date.now() < endTime) { + await this.doodlebot.sendWebsocketCommand("d,O"); + await new Promise((res) => setTimeout(res, interval)); + await this.doodlebot.sendWebsocketCommand("d,N"); + await new Promise((res) => setTimeout(res, interval)); + } - const textResponse = response.headers.get("text-response"); - console.log("Text Response:", textResponse); - - const blob = await response.blob(); - const audioUrl = URL.createObjectURL(blob); - console.log("Audio URL:", audioUrl); - - const audio = new Audio(audioUrl); - const array = await blob.arrayBuffer(); - uint8array = new Uint8Array(array); - // } else { - // const base64 = await this.blobToBase64(blob); - // const payload = { - // filename: file.name, - // content: base64, - // mimeType: blob.type, - // }; - // response = await this.doodlebot.fetch(endpoint, "chatgpt", payload); - // uint8array = new Uint8Array([...atob(response)].map(char => char.charCodeAt(0))); - // } - const interval = 50; // 0.2 seconds in milliseconds - const endTime = Date.now() + 1 * 1000; - - this.doodlebot.sendAudioData(uint8array); - while (Date.now() < endTime) { - await this.doodlebot.sendWebsocketCommand("d,O"); - await new Promise((res) => setTimeout(res, interval)); - await this.doodlebot.sendWebsocketCommand("d,N"); - await new Promise((res) => setTimeout(res, interval)); - } - } catch (error) { - console.error("Error sending audio file:", error); + console.error("Error sending audio file:", error); } } async isValidWavFile(file) { const arrayBuffer = await file.arrayBuffer(); const dataView = new DataView(arrayBuffer); - + // Check the "RIFF" chunk descriptor const riff = String.fromCharCode(...new Uint8Array(arrayBuffer.slice(0, 4))); if (riff !== "RIFF") { console.error("Invalid WAV file: Missing RIFF header"); return false; } - + // Check the "WAVE" format const wave = String.fromCharCode(...new Uint8Array(arrayBuffer.slice(8, 12))); if (wave !== "WAVE") { console.error("Invalid WAV file: Missing WAVE format"); return false; } - + // Check for "fmt " subchunk const fmt = String.fromCharCode(...new Uint8Array(arrayBuffer.slice(12, 16))); if (fmt !== "fmt ") { console.error("Invalid WAV file: Missing fmt subchunk"); return false; } - + // Check for "data" subchunk const dataIndex = arrayBuffer.byteLength - 8; // Approximate location const dataChunk = String.fromCharCode(...new Uint8Array(arrayBuffer.slice(dataIndex, dataIndex + 4))); @@ -1574,39 +1376,39 @@ blobToBase64(blob) { console.error("Invalid WAV file: Missing data subchunk"); return false; } - + console.log("Valid WAV file"); return true; } - + async processAndSendAudio(buffer, endpoint, seconds) { try { - const wavBlob = await this.saveAudioBufferToWav(buffer); - console.log(wavBlob); - const wavFile = new File([wavBlob], "output.wav", { type: "audio/wav" }); - - // // Create a temporary URL for the file - // const url = URL.createObjectURL(wavFile); - - // // Create a temporary anchor element - // const a = document.createElement("a"); - // a.href = url; - // a.download = "output.wav"; - // document.body.appendChild(a); - - // // Trigger the download - // a.click(); - - // // Clean up - // document.body.removeChild(a); - // URL.revokeObjectURL(url); - // const isValid = await this.isValidWavFile(wavFile); - // if (!isValid) { - // throw new Error("Generated file is not a valid WAV file"); - // } - await this.sendAudioFileToChatEndpoint(wavFile, endpoint, wavBlob, seconds); + const wavBlob = await this.saveAudioBufferToWav(buffer); + console.log(wavBlob); + const wavFile = new File([wavBlob], "output.wav", { type: "audio/wav" }); + + // // Create a temporary URL for the file + // const url = URL.createObjectURL(wavFile); + + // // Create a temporary anchor element + // const a = document.createElement("a"); + // a.href = url; + // a.download = "output.wav"; + // document.body.appendChild(a); + + // // Trigger the download + // a.click(); + + // // Clean up + // document.body.removeChild(a); + // URL.revokeObjectURL(url); + // const isValid = await this.isValidWavFile(wavFile); + // if (!isValid) { + // throw new Error("Generated file is not a valid WAV file"); + // } + await this.sendAudioFileToChatEndpoint(wavFile, endpoint, wavBlob, seconds); } catch (error) { - console.error("Error processing and sending audio:", error); + console.error("Error processing and sending audio:", error); } } @@ -1619,18 +1421,18 @@ blobToBase64(blob) { console.log("recording audio?") const { context, buffer } = await this.doodlebot?.recordAudio(seconds); console.log("finished recording audio"); - + // Display "thinking" while processing and waiting for response await this.doodlebot?.display("clear"); await this.doodlebot?.displayText("thinking"); - + // Before sending audio to be played await this.processAndSendAudio(buffer, endpoint, seconds); - + // Display "speaking" when ready to speak // await this.doodlebot?.display("clear"); // await this.doodlebot?.displayText("speaking"); - + // Wait a moment before clearing the display await new Promise(resolve => setTimeout(resolve, 2000)); await this.doodlebot?.display("h"); @@ -1640,28 +1442,28 @@ blobToBase64(blob) { try { const modelUrl = this.modelArgumentToURL(url); console.log('Loading model from URL:', modelUrl); - + // Initialize prediction state if needed this.predictionState[modelUrl] = {}; - + // Load and initialize the model const { model, type } = await this.initModel(modelUrl); this.predictionState[modelUrl].modelType = type; this.predictionState[modelUrl].model = model; - + // Update the current model reference this.teachableImageModel = modelUrl; - - await this.indicate({ - type: "success", - msg: "Model loaded successfully" + + await this.indicate({ + type: "success", + msg: "Model loaded successfully" }); } catch (e) { console.error('Error loading model:', e); this.teachableImageModel = null; - await this.indicate({ - type: "error", - msg: "Failed to load model" + await this.indicate({ + type: "error", + msg: "Failed to load model" }); } } @@ -1670,7 +1472,7 @@ blobToBase64(blob) { // Convert user-provided model URL/ID to the correct format const endpointProvidedFromInterface = "https://teachablemachine.withgoogle.com/models/"; const redirectEndpoint = "https://storage.googleapis.com/tm-model/"; - + return modelArg.startsWith(endpointProvidedFromInterface) ? modelArg.replace(endpointProvidedFromInterface, redirectEndpoint) : redirectEndpoint + modelArg + "/"; @@ -1684,12 +1486,12 @@ blobToBase64(blob) { // First try loading as an image model try { const customMobileNet = await tmImage.load(modelURL, metadataURL); - + // Check if it's actually an audio model if ((customMobileNet as any)._metadata.hasOwnProperty('tfjsSpeechCommandsVersion')) { const recognizer = await speechCommands.create("BROWSER_FFT", undefined, modelURL, metadataURL); await recognizer.ensureModelLoaded(); - + // Setup audio listening await recognizer.listen(async result => { this.latestAudioResults = result; @@ -1699,9 +1501,9 @@ blobToBase64(blob) { invokeCallbackOnNoiseAndUnknown: true, overlapFactor: 0.50 }); - + return { model: recognizer, type: this.ModelType.AUDIO }; - } + } // Check if it's a pose model else if ((customMobileNet as any)._metadata.packageName === "@teachablemachine/pose") { const customPoseNet = await tmPose.load(modelURL, metadataURL); @@ -1795,7 +1597,7 @@ blobToBase64(blob) { try { const imageStream = this.getImageStream(); if (!imageStream) { - console.error("Failed to get image stream"); + //console.error("Failed to get image stream"); return; } // const imageBitmap = await createImageBitmap(imageStream); @@ -1880,21 +1682,21 @@ blobToBase64(blob) { } // Create indicator to show progress - const indicator = await this.indicate({ + const indicator = await this.indicate({ type: "warning", - msg: `Capturing snapshots of ${imageClass}...` + msg: `Capturing snapshots of ${imageClass}...` }); const snapshots: string[] = []; const zip = new JSZip(); - + // Ensure we have video stream this.imageStream ??= this.doodlebot?.getImageStream(); if (!this.imageStream) { indicator.close(); - await this.indicate({ - type: "error", - msg: "No video stream available" + await this.indicate({ + type: "error", + msg: "No video stream available" }); return; } @@ -1902,50 +1704,50 @@ blobToBase64(blob) { // Capture a snapshot every 500ms const interval = 100; // 500ms between snapshots const iterations = (seconds * 1000) / interval; - + for (let i = 0; i < iterations; i++) { // Create a canvas to draw the current frame const canvas = document.createElement('canvas'); canvas.width = this.imageStream.width; canvas.height = this.imageStream.height; const ctx = canvas.getContext('2d'); - + // Draw current frame to canvas ctx.drawImage(this.imageStream, 0, 0); - + // Convert to base64 and store const dataUrl = canvas.toDataURL('image/jpeg'); snapshots.push(dataUrl); - + // Add to zip file const base64Data = dataUrl.replace(/^data:image\/jpeg;base64,/, ""); - zip.file(`${imageClass}_${i+1}.jpg`, base64Data, {base64: true}); - + zip.file(`${imageClass}_${i + 1}.jpg`, base64Data, { base64: true }); + // Wait for next interval await new Promise(resolve => setTimeout(resolve, interval)); } // Generate zip file - const content = await zip.generateAsync({type: "blob"}); - + const content = await zip.generateAsync({ type: "blob" }); + // Create download link with image class in filename const downloadUrl = URL.createObjectURL(content); const link = document.createElement('a'); link.href = downloadUrl; link.download = `${imageClass}_snapshots.zip`; - + // Trigger download document.body.appendChild(link); link.click(); document.body.removeChild(link); - + // Cleanup URL.revokeObjectURL(downloadUrl); indicator.close(); - - await this.indicate({ - type: "success", - msg: `Captured ${snapshots.length} snapshots of ${imageClass}` + + await this.indicate({ + type: "success", + msg: `Captured ${snapshots.length} snapshots of ${imageClass}` }); if (this.SOCIAL && Math.random() < this.socialness) { @@ -1986,9 +1788,9 @@ blobToBase64(blob) { resolve(null); }); }); - + const durationMs = audio.duration * 1000; // duration is in seconds - + // Convert blob to Uint8Array and send to Doodlebot const array = await blob.arrayBuffer(); await this.doodlebot.sendAudioData(new Uint8Array(array)); diff --git a/extensions/src/doodlebot/package.json b/extensions/src/doodlebot/package.json index 3f9121176..3ef8e11ac 100644 --- a/extensions/src/doodlebot/package.json +++ b/extensions/src/doodlebot/package.json @@ -24,6 +24,7 @@ "canvas": "^2.11.2", "cubic-spline": "^3.0.3", "curve-matcher": "^1.1.1", + "eventemitter3": "^5.0.1", "events": "^3.3.0", "jszip": "^3.10.1" } diff --git a/extensions/src/doodlebot/pnpm-lock.yaml b/extensions/src/doodlebot/pnpm-lock.yaml index bd4c89c00..a443d7fd1 100644 --- a/extensions/src/doodlebot/pnpm-lock.yaml +++ b/extensions/src/doodlebot/pnpm-lock.yaml @@ -41,6 +41,9 @@ importers: curve-matcher: specifier: ^1.1.1 version: 1.1.1 + eventemitter3: + specifier: ^5.0.1 + version: 5.0.1 events: specifier: ^3.3.0 version: 3.3.0 @@ -265,6 +268,9 @@ packages: resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} engines: {node: '>=6'} + eventemitter3@5.0.1: + resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} + events@3.3.0: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} @@ -786,6 +792,8 @@ snapshots: escalade@3.2.0: {} + eventemitter3@5.0.1: {} + events@3.3.0: {} follow-redirects@1.15.9: {} diff --git a/extensions/src/doodlebot/utils.ts b/extensions/src/doodlebot/utils.ts index e27473cdf..58ca8f327 100644 --- a/extensions/src/doodlebot/utils.ts +++ b/extensions/src/doodlebot/utils.ts @@ -27,7 +27,7 @@ export const base64ToInt32Array = async (base64) => { export const makeWebsocket = (ip: string, path: string) => new WebSocket(`wss://${ip}${path}`); export const testWebSocket = (ip: string, port: string | number, timeoutSeconds?: number) => { - const websocket = makeWebsocket(ip, port); + const websocket = makeWebsocket(ip, String(port)); return new Promise((resolve) => { websocket.onopen = () => websocket.close(); websocket.onclose = (event) => resolve(event.wasClean); @@ -36,3 +36,18 @@ export const testWebSocket = (ip: string, port: string | number, timeoutSeconds? } export const Max32Int = 2147483647; + + +export const deferred = () => { + let resolve: (value: T | PromiseLike) => void; + let reject: (reason?: any) => void; + + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + + return { promise, resolve: resolve!, reject: reject! }; +}; + +export type Deferred = ReturnType>; \ No newline at end of file diff --git a/scratch-packages/scratch-gui b/scratch-packages/scratch-gui index 5585d5eca..91c188acf 160000 --- a/scratch-packages/scratch-gui +++ b/scratch-packages/scratch-gui @@ -1 +1 @@ -Subproject commit 5585d5ecaeb0bcd50b4efabeb84811c924c17a1f +Subproject commit 91c188acf6f2f1a9e267c8a5a61e196fb18bcc39 diff --git a/scratch-packages/scratch-vm b/scratch-packages/scratch-vm index d79e45862..6b355943a 160000 --- a/scratch-packages/scratch-vm +++ b/scratch-packages/scratch-vm @@ -1 +1 @@ -Subproject commit d79e45862488335382fb03a2bb27032154c60f73 +Subproject commit 6b355943a631e17a2c9d37b4df3ea53a11506568