diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
new file mode 100644
index 000000000..510bf730d
--- /dev/null
+++ b/.devcontainer/devcontainer.json
@@ -0,0 +1,30 @@
+{
+ "postStartCommand": "pnpm config set store-dir /home/vscode/.local/share/pnpm/store && pnpm install",
+ "customizations": {
+ "vscode": {
+ "extensions": [
+ "ms-vscode.vscode-typescript-next",
+ "esbenp.prettier-vscode",
+ "svelte.svelte-vscode"
+ ],
+ "settings": {
+ "[svelte]": {
+ "editor.defaultFormatter": "svelte.svelte-vscode"
+ },
+ "[typescript]": {
+ "editor.defaultFormatter": "esbenp.prettier-vscode"
+ },
+ "editor.formatOnSave": true,
+ "editor.tabSize": 2,
+ "svelte.enable-ts-plugin": true
+ }
+ }
+ },
+ "features": {
+ "ghcr.io/devcontainers/features/node:latest": {
+ "version": "20"
+ },
+ "ghcr.io/pmalacho-mit/devcontainer-features/git-subrepo:latest": {}
+ },
+ "image": "mcr.microsoft.com/devcontainers/base:ubuntu-24.04"
+}
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 06b67e472..c97711554 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
+**/.pnpm-store
.DS_Store
./**/.DS_Store
diff --git a/extensions/src/doodlebot/Connect.svelte b/extensions/src/doodlebot/Connect.svelte
index 92ff1f8f8..ef1fe2b95 100644
--- a/extensions/src/doodlebot/Connect.svelte
+++ b/extensions/src/doodlebot/Connect.svelte
@@ -1,7 +1,11 @@
- {#if error}
-
- {error}
+ {#if connected}
+
You're connected to doodlebot!
+
+ If you'd like to reconnect, or connect to a different device, you must
+ reload this page.
- {/if}
- {#if bluetooth}
-
How to connect to doodlebot
-
1. Set network credentials:
-
- SSID (Network Name):
-
-
-
- Password:
-
-
+
+
+ {:else}
+ {#if error}
+
+ {error}
+
+ {/if}
+ {#if bluetooth}
+
Please connect to a doodlebot...
+
+
...by selecting a bluetooth device
+
+ {#if bleDevice}
+ You've selected 🤖 {bleDevice.device.name}.
+ {/if}
+
+
+
+
-
-
-
2. Select bluetooth device
-
-
-
- {:else}
- Uh oh! Your browser does not support bluetooth. Here's how to fix that...
- TBD
+ {:else}
+ Uh oh! Your browser does not support bluetooth. Please contact an
+ instructor.
+ {/if}
{/if}
@@ -165,7 +163,70 @@
outline: none;
}
- .ip {
- width: 3rem;
+ .open {
+ background-color: dodgerblue;
+ border: 1px solid dodgerblue;
+ border-radius: 4px;
+ box-shadow: rgba(0, 0, 0, 0.1) 0 2px 4px 0;
+ box-sizing: border-box;
+ color: #fff;
+ cursor: pointer;
+ font-family:
+ "Akzidenz Grotesk BQ Medium",
+ -apple-system,
+ BlinkMacSystemFont,
+ sans-serif;
+ font-size: 16px;
+ font-weight: 600;
+ outline: none;
+ outline: 0;
+ padding: 5px 15px;
+ text-align: center;
+ transform: translateY(0);
+ transition:
+ transform 150ms,
+ box-shadow 150ms;
+ user-select: none;
+ -webkit-user-select: none;
+ touch-action: manipulation;
+ }
+
+ /* CSS */
+ .connect {
+ background-color: #13aa52;
+ border: 1px solid #13aa52;
+ border-radius: 4px;
+ box-shadow: rgba(0, 0, 0, 0.1) 0 2px 4px 0;
+ box-sizing: border-box;
+ color: #fff;
+ cursor: pointer;
+ font-family:
+ "Akzidenz Grotesk BQ Medium",
+ -apple-system,
+ BlinkMacSystemFont,
+ sans-serif;
+ font-size: 16px;
+ font-weight: 600;
+ outline: none;
+ outline: 0;
+ padding: 10px 25px;
+ text-align: center;
+ transform: translateY(0);
+ transition:
+ transform 150ms,
+ box-shadow 150ms;
+ user-select: none;
+ -webkit-user-select: none;
+ touch-action: manipulation;
+ }
+
+ .connect:disabled {
+ opacity: 0.5;
+ cursor: not-allowed;
+ }
+
+ .connect:not(:disabled):hover {
+ box-shadow: rgba(0, 0, 0, 0.15) 0 3px 9px 0;
+ transform: translateY(-2px);
}
diff --git a/extensions/src/doodlebot/Doodlebot.ts b/extensions/src/doodlebot/Doodlebot.ts
index f5e8f5600..efe612b8a 100644
--- a/extensions/src/doodlebot/Doodlebot.ts
+++ b/extensions/src/doodlebot/Doodlebot.ts
@@ -1,13 +1,14 @@
-import EventEmitter from "events";
-import { Service } from "./communication/ServiceHelper";
+///
+
+import { EventEmitter } from "eventemitter3";
import UartService from "./communication/UartService";
import { followLine } from "./LineFollowing";
import { Command, DisplayKey, NetworkStatus, ReceivedCommand, SensorKey, command, display, endpoint, keyBySensor, motorCommandReceived, networkStatus, port, sensor } from "./enums";
-import { base64ToInt32Array, makeWebsocket, Max32Int, testWebSocket } from "./utils";
+import { base64ToInt32Array, deferred, makeWebsocket, Max32Int } from "./utils";
import { LineDetector } from "./LineDetection";
import { calculateArcTime } from "./TimeHelper";
+import type { BLEDeviceWithUartService } from "./ble";
-export type Services = Awaited>;
export type MotorStepRequest = {
/** Number of steps for the stepper (+ == forward, - == reverse) */
steps: number,
@@ -19,14 +20,10 @@ export type Vector3D = { x: number, y: number, z: number };
export type Color = { red: number, green: number, blue: number, alpha: number };
export type SensorReading = number | Vector3D | Bumper | Color;
export type SensorData = Doodlebot["sensorData"];
-export type NetworkCredentials = { ssid: string, password: string, ipOverride?: string };
-export type NetworkConnection = { ip: string, hostname?: string };
-export type RequestBluetooth = (callback: (bluetooth: Bluetooth) => any) => void;
-export type SaveIP = (ip: string) => void;
type MaybePromise = undefined | Promise;
-type Pending = Record<"motor" | "wifi" | "websocket" | "video" | "image", MaybePromise> & { ip: MaybePromise };
+type Pending = Record<"motor" | "websocket" | "video" | "image", MaybePromise>;
type SubscriptionTarget = Pick;
@@ -40,134 +37,33 @@ type MotorCommand = "steps" | "arc" | "stop";
const trimNewtworkStatusMessage = (message: string, prefix: NetworkStatus) => message.replace(prefix, "").trim();
-const localIp = "127.0.0.1";
-
const events = {
stop: "motor",
connect: "connect",
disconnect: "disconnect",
} as const;
-type CreatePayload = {
- credentials: NetworkCredentials,
- requestBluetooth: RequestBluetooth,
- saveIP: SaveIP,
-}
-
-const msg = (content: string, type: "success" | "warning" | "error") => {
- switch (type) {
- case "success":
- console.log(content);
- break;
- case "warning":
- console.warn(content);
- break;
- case "error":
- console.error(content);
- break;
- }
-}
-
type BLECommunication = {
onDisconnect: (...callbacks: (() => void)[]) => void,
onReceive: (callback: (text: CustomEvent) => void) => void,
send: (text: string) => Promise
}
-export default class Doodlebot {
- /**
- *
- * @param services
- * @param serviceClass
- * @returns
- */
- static async tryCreateService any)>(
- services: BluetoothRemoteGATTService[], serviceClass: T
- ): Promise> {
- const found = services.find((service) => service.uuid === serviceClass.uuid);
- return found ? await serviceClass.create(found) : undefined;
- }
-
- /**
- *
- * @param bluetooth
- * @param devicePrefix @todo unused
- * @returns
- */
- static async requestRobot(bluetooth: Bluetooth, ...filters: BluetoothLEScanFilter[]) {
- const device = await bluetooth.requestDevice({
- filters: [
- ...(filters ?? []),
- {
- services: [UartService.uuid]
- },
- ],
- });
-
- return device;
- }
-
- /**
- * Get
- * @param device
- * @returns
- */
- static async getServices(device: BluetoothDevice) {
- if (!device || !device.gatt) return null;
- if (!device.gatt.connected) await device.gatt.connect();
- const services = await device.gatt.getPrimaryServices();
- const uartService = await Doodlebot.tryCreateService(services, UartService);
-
- return { uartService, };
- }
-
- static async getBLE(ble: Bluetooth, ...filters: BluetoothLEScanFilter[]) {
- const robot = await Doodlebot.requestRobot(ble, ...filters);
- const services = await Doodlebot.getServices(robot);
- if (!services) throw new Error("Unable to connect to doodlebot's UART service");
- return { robot, services };
- }
-
- /**
- *
- * @param ble
- * @param filters
- * @throws
- * @returns
- */
- static async tryCreate(
- ble: Bluetooth,
- { requestBluetooth, credentials, saveIP }: CreatePayload,
- ...filters: BluetoothLEScanFilter[]) {
- const { robot, services } = await Doodlebot.getBLE(ble, ...filters);
- return new Doodlebot({
- onReceive: (callback) => services.uartService.addEventListener("receiveText", callback),
- onDisconnect: (callback) => ble.addEventListener("gattserverdisconnected", callback),
- send: (text) => services.uartService.sendText(text),
- }, requestBluetooth, credentials, saveIP, async (description) => {
- const response = await fetch(`http://192.168.41.214:8001/webrtc`, {
- method: 'POST',
- body: description,
- headers: { 'Content-Type': 'application/json' }
- });
- console.log("INSIDE INTERNAL FUNCTION");
- const responseJson = await response.json();
- console.log("kjson", responseJson);
- return responseJson;
- });
- }
-
- private pending: Pending = { motor: undefined, wifi: undefined, websocket: undefined, ip: undefined };
+export default class Doodlebot {
+ private pending: Pending = { motor: undefined, websocket: undefined, video: undefined, image: undefined };
private onMotor = new EventEmitter();
private onSensor = new EventEmitter();
- private onNetwork = new EventEmitter();
private disconnectCallbacks = new Set<() => void>();
private subscriptions = new Array>();
- private connection: NetworkConnection;
private websocket: WebSocket;
private encoder = new TextEncoder();
+ public readonly topLevelDomain = deferred();
+ public readonly bleDevice = deferred();
+
+ private readonly ble = deferred();
+
private lastDisplayedKey;
private lastDisplayedType;
@@ -289,39 +185,44 @@ export default class Doodlebot {
private audioSocket: WebSocket;
private audioCallbacks = new Set<(chunk: Float32Array) => void>();
- private pc: any;
- private webrtcVideo: any;
+ private pc: RTCPeerConnection;
+ private webrtcVideo: HTMLVideoElement;
public previewImage;
public canvasWebrtc;
- constructor(
- private ble: BLECommunication,
- private requestBluetooth: RequestBluetooth,
- private credentials: NetworkCredentials,
- private saveIP: SaveIP,
- private fetchFunction
- ) {
- this.ble.onReceive(this.receiveTextBLE.bind(this));
- this.ble.onDisconnect(this.handleBleDisconnect.bind(this));
- this.connectionWorkflow(credentials);
-
+ private reloadRequired?: ((msg: string) => void) | null = null;
+
+ constructor() {
+ this.ble.promise.then(ble => ble.onReceive(this.receiveTextBLE.bind(this)));
+ this.ble.promise.then(ble => ble.onDisconnect(this.handleBleDisconnect.bind(this)));
+ this.bleDevice.promise.then(async ({ device, service }) => {
+ this.ble.resolve(
+ {
+ onReceive: (callback) => service.addEventListener("receiveText", callback),
+ onDisconnect: (callback) => device.addEventListener("gattserverdisconnected", callback),
+ send: (text) => service.sendText(text),
+ });
+ })
+
+ this.connectionWorkflow();
+
this.pc = new RTCPeerConnection();
this.pc.addTransceiver("video", { direction: "recvonly" });
console.log("pc", this.pc);
-
+
this.pc.ontrack = (event) => {
const stream = event.streams[0];
console.log("stream", stream);
-
+
this.webrtcVideo = document.createElement('video');
this.webrtcVideo.srcObject = stream;
this.webrtcVideo.autoplay = true;
this.webrtcVideo.playsInline = true;
// document.body.appendChild(this.webrtcVideo);
-
+
this.webrtcVideo.play().catch(e => console.error("Playback error:", e));
-
+
this.webrtcVideo.onerror = (error) => {
console.log("ERROR 2", error);
};
@@ -345,39 +246,26 @@ export default class Doodlebot {
};
this.webrtcVideo.requestVideoFrameCallback(handleVideoFrame);
};
-
- // this is an ugly hack just to get things working
- // (it was crashing on my machine)
- // the pc.createOffer() call below needs the IP
- // but this is before the IP address has been prompted
- // for. I think it works sometimes because we save the
- // address for next time, but fails the first time
- // it looks like this ps.createOffer call was just chucked
- // here in the constructor as a quick test of webrtc video?
- // -jon
- const urlParams = new URLSearchParams(window.location.search);
- const ip = urlParams.get("ip");
+
this.pc.createOffer()
.then(offer => this.pc.setLocalDescription(offer))
- //.then(() => fetch(`http://192.168.41.231:8001/webrtc`, {
- .then(() => fetch(`https://${ip}/api/v1/video/webrtc`, {
- method: 'POST',
- body: JSON.stringify(this.pc.localDescription),
- headers: { 'Content-Type': 'application/json' }
- }))
+ .then(async () => {
+ const tld = await this.topLevelDomain.promise;
+ console.log("TLD", tld);
+ const webrtcResponse = await fetch(`https://${tld}/api/v1/video/webrtc`, {
+ method: 'POST',
+ body: JSON.stringify(this.pc.localDescription),
+ headers: { 'Content-Type': 'application/json' }
+ });
+ console.log("webrtcResponse", webrtcResponse);
+ return webrtcResponse;
+ })
.then(response => response.json())
.then(answer => this.pc.setRemoteDescription(answer))
.catch(err => console.error("WebRTC error:", err));
-
+
this.previewImage = document.createElement("img");
document.body.appendChild(this.previewImage);
-
- // fetch(`http://192.168.41.214:8000/webrtc`, {
- // method: 'POST',
- // body: JSON.stringify(this.pc.localDescription),
- // headers: { 'Content-Type': 'application/json' }
- // })
-
}
private formCommand(...args: (string | number)[]) {
@@ -401,11 +289,7 @@ export default class Doodlebot {
private updateNetworkStatus(ipComponent: string, hostnameComponent: string) {
const ip = trimNewtworkStatusMessage(ipComponent, networkStatus.ipPrefix);
const hostname = trimNewtworkStatusMessage(hostnameComponent, networkStatus.hostnamePrefix);
- if (ip === localIp) {
- return this.onNetwork.emit(events.disconnect);
- }
- this.connection = { ip, hostname };
- this.onNetwork.emit(events.connect, this.connection);
+ this.reloadRequired?.("Network settings changed, please reload the page and reconnect to doodlebot.");
}
private receiveTextBLE(event: CustomEvent) {
@@ -452,7 +336,7 @@ export default class Doodlebot {
const [x, y, z] = parameters.map((parameter) => Number.parseFloat(parameter));
this.updateSensor(keyBySensor[command], { x, y, z });
break;
-
+
}
case sensor.light: {
const [red, green, blue, alpha] = parameters.map((parameter) => Number.parseFloat(parameter));
@@ -476,19 +360,10 @@ export default class Doodlebot {
const decodedMessage = decoder.decode(event.data);
console.log('Received ArrayBuffer as text:', decodedMessage);
}
-
- }
-
- private invalidateWifiConnection() {
- this.connection = undefined;
- this.pending.wifi = undefined;
- this.pending.websocket = undefined;
- this.websocket?.close();
- this.websocket = undefined;
}
private handleBleDisconnect() {
- console.log("disconnected!!!");
+ this.reloadRequired?.("Doodlebot bluetooth disconnected, please reload the page and reconnect to doodlebot.");
for (const callback of this.disconnectCallbacks) callback();
for (const { target, event, listener } of this.subscriptions) target.removeEventListener(event, listener);
}
@@ -529,14 +404,14 @@ export default class Doodlebot {
scheduleDisableSensor(type: SensorKey, delay = 5000) {
// Reset existing timer
if (this.disableTimers[type]) {
- clearTimeout(this.disableTimers[type]);
+ clearTimeout(this.disableTimers[type]);
}
-
+
this.disableTimers[type] = setTimeout(() => {
- this.disableSensor(type);
- delete this.disableTimers[type];
+ this.disableSensor(type);
+ delete this.disableTimers[type];
}, delay);
- }
+ }
/**
*
@@ -545,25 +420,25 @@ export default class Doodlebot {
*/
async getSensorReading(type: T): Promise {
await this.enableSensor(type);
-
+
const reading = this.sensorData[type];
-
+
// Schedule auto-disable after 5s of inactivity
this.scheduleDisableSensor(type, 5000);
-
+
return reading;
}
async getSingleSensorReading(
type: T
- ): Promise {
+ ): Promise {
await this.enableSensor(type);
-
+
const reading = this.sensorData[type];
-
+
// Schedule auto-disable after 5s of inactivity
this.scheduleDisableSensor(type, 5000);
-
+
return reading;
}
@@ -620,10 +495,8 @@ export default class Doodlebot {
}
async findImageFiles() {
- while (!this.connection) {
- await new Promise(resolve => setTimeout(resolve, 100));
- }
- let endpoint = "https://" + this.connection.ip + "/api/v1/upload/images"
+ const tld = await this.topLevelDomain.promise;
+ let endpoint = "https://" + tld + "/api/v1/upload/images"
let uploadedImages = await this.fetchAndExtractList(endpoint);
return uploadedImages.filter(item => !this.imageFiles.includes(item));
}
@@ -632,7 +505,7 @@ export default class Doodlebot {
private streamActive: Record = {};
private stopTimers: Record = {};
- async getFacePrediction(ip: string, type: "face" | "object") {
+ async getFacePrediction(type: "face" | "object") {
const now = Date.now();
// Clear old stop timers whenever we get a new call
@@ -643,57 +516,56 @@ export default class Doodlebot {
// If stream is already active → just return continuous predict
if (this.streamActive[type]) {
this.lastCallTime[type] = now;
- this.resetStopTimer(ip, type);
- return await this.getContinuousPredict(ip, type);
+ this.resetStopTimer(type);
+ return await this.getContinuousPredict(type);
}
// If first call or more than 1s since last call → single_predict
if (!this.lastCallTime[type] || now - this.lastCallTime[type] > 1000) {
this.lastCallTime[type] = now;
console.log(`[${type}] Calling single_predict API...`);
- return await this.callSinglePredict(ip);
+ return await this.callSinglePredict();
}
// If called again within 1s → switch to stream
console.log(`[${type}] Switching to stream API...`);
this.streamActive[type] = true;
this.lastCallTime[type] = now;
- await this.startContinuousDetection(ip, type);
- this.resetStopTimer(ip, type);
- return await this.getContinuousPredict(ip, type);
+ await this.startContinuousDetection(type);
+ this.resetStopTimer(type);
+ return await this.getContinuousPredict(type);
}
- private resetStopTimer(ip: string, type: "face" | "object") {
+ private resetStopTimer(type: "face" | "object") {
this.stopTimers[type] = setTimeout(() => {
console.log(`[${type}] No calls in 5s, stopping stream...`);
- this.stopContinuousDetection(ip, type);
+ this.stopContinuousDetection(type);
}, 5000);
}
- async callSinglePredict(ip: string) {
- const uploadEndpoint = `https://${ip}/api/v1/video/single_predict?width=320&height=240`;
+ async callSinglePredict() {
+ const tld = await this.topLevelDomain.promise;
+ const uploadEndpoint = `https://${tld}/api/v1/video/single_predict?width=320&height=240`;
const response = await fetch(uploadEndpoint);
return await response.json();
}
- async startContinuousDetection(ip: string, type: "face" | "object") {
- while (!this.connection) {
- await new Promise(resolve => setTimeout(resolve, 100));
- }
+ async startContinuousDetection(type: "face" | "object") {
+ const tld = await this.topLevelDomain.promise;
let endpoint;
if (this.streamActive["face"] && this.streamActive["object"]) {
- endpoint = `https://${ip}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=true&set_detect_faces=true`;
+ endpoint = `https://${tld}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=true&set_detect_faces=true`;
} else if (this.streamActive["face"]) {
- endpoint = `https://${ip}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=false&set_detect_faces=true`;
+ endpoint = `https://${tld}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=false&set_detect_faces=true`;
} else {
- endpoint = `https://${ip}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=true&set_detect_faces=false`;
+ endpoint = `https://${tld}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=true&set_detect_faces=false`;
}
console.log("starting", endpoint);
await fetch(endpoint);
console.log(`[${type}] Continuous detection started`);
}
- async stopContinuousDetection(ip: string, type: "face" | "object") {
+ async stopContinuousDetection(type: "face" | "object") {
this.streamActive[type] = false;
this.lastCallTime[type] = 0;
if (this.stopTimers[type]) {
@@ -701,33 +573,33 @@ export default class Doodlebot {
delete this.stopTimers[type];
}
let endpoint;
+ const tld = await this.topLevelDomain.promise;
if (!this.streamActive["face"] && !this.streamActive["object"]) {
- endpoint = `https://${ip}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=false&set_detect_faces=false`;
- } else if (!this.streamActive["face"] ) {
- endpoint = `https://${ip}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=true&set_detect_faces=false`;
+ endpoint = `https://${tld}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=false&set_detect_faces=false`;
+ } else if (!this.streamActive["face"]) {
+ endpoint = `https://${tld}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=true&set_detect_faces=false`;
} else {
- endpoint = `https://${ip}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=false&set_detect_faces=true`;
+ endpoint = `https://${tld}/api/v1/video/stream?width=640&height=480&set_display=true&set_detect_objects=false&set_detect_faces=true`;
}
console.log("stopping", endpoint)
await fetch(endpoint);
console.log(`[${type}] Continuous detection stopped`);
}
- async getContinuousPredict(ip: string, type: "face" | "object") {
- while (!this.connection) {
- await new Promise(resolve => setTimeout(resolve, 100));
+ async getContinuousPredict(type: "face" | "object") {
+ const tld = await this.topLevelDomain.promise;
+
+ const endpoint = `https://${tld}/api/v1/video/stream_latest?screenshot=false`;
+
+ const response = await fetch(endpoint);
+ if (!response.ok) {
+ await this.startContinuousDetection(type);
+ return this.getContinuousPredict(type);
+ } else {
+ return await response.json();
}
- const endpoint = `https://${ip}/api/v1/video/stream_latest?screenshot=false`;
- const response = await fetch(endpoint);
- if (!response.ok) {
- await this.startContinuousDetection(ip, type);
- return this.getContinuousPredict(ip, type);
- } else {
- return await response.json();
- }
-
}
getReadingLocation(axis, type, reading) {
@@ -748,15 +620,15 @@ export default class Doodlebot {
const firstApple = reading.objects.find(obj => obj.label === "apple");
if (firstApple) {
if (axis == "x") {
- return firstApple.x;
+ return firstApple.x;
} else {
- return firstApple.y;
+ return firstApple.y;
}
} else {
return -1;
}
} else {
- const firstOrange = reading.objects.find(obj => obj.label === "orange");
+ const firstOrange = reading.objects.find(obj => obj.label === "orange");
if (firstOrange) {
if (axis == "x") {
return firstOrange.x;
@@ -766,16 +638,13 @@ export default class Doodlebot {
} else {
return -1;
}
- }
+ }
}
}
async findSoundFiles() {
- while (!this.connection) {
- await new Promise(resolve => setTimeout(resolve, 100));
- }
- if (!this.connection) return [];
- let endpoint = "https://" + this.connection.ip + "/api/v1/upload/sounds"
+ const tld = await this.topLevelDomain.promise;
+ let endpoint = "https://" + tld + "/api/v1/upload/sounds"
let uploadedSounds = await this.fetchAndExtractList(endpoint);
return uploadedSounds.filter(item => !this.soundFiles.includes(item));
}
@@ -840,9 +709,9 @@ export default class Doodlebot {
case "stop":
if (this.isStopped) return;
await this.sendBLECommand(command.motor, "s");
- // return await this.untilFinishedPending("motor", new Promise(async (resolve) => {
- // this.onMotor.once(events.stop, resolve);
- // }));
+ // return await this.untilFinishedPending("motor", new Promise(async (resolve) => {
+ // this.onMotor.once(events.stop, resolve);
+ // }));
}
}
@@ -866,124 +735,13 @@ export default class Doodlebot {
await this.sendBLECommand(command.lowPower);
}
- async getIPAddress() {
- console.log(this.connection);
- if (this.connection && this.connection.ip) {
- console.log("returning")
- return this.connection.ip;
- }
- const self = this;
- const interval = setTimeout(() => this.sendBLECommand(command.network), 1000);
- const ip = await new Promise(async (resolve) => {
- this.onNetwork.once(events.connect, () => resolve(self.connection.ip));
- this.onNetwork.once(events.disconnect, () => resolve(null));
- });
- console.log(`Got ip: ${ip}`);
- clearTimeout(interval);
- return ip;
- }
-
- async testIP(ip: string) {
- if (!ip) return false;
- const controller = new AbortController();
- const timeout = setTimeout(() => controller.abort(), 2000);
- try {
- const resp = await fetch(`http://${ip}:8000/static/webrtc-client.html`, { signal: controller.signal });
- return resp.ok;
- }
- catch {
- return false;
- }
- finally {
- clearTimeout(timeout);
- }
- }
-
- setIP(ip: string) {
- this.connection ??= { ip };
- this.saveIP(ip);
- return this.connection.ip = ip;
- }
-
- getStoredIPAddress() {
- if (!this.connection) { return "" }
- return this.connection.ip;
- }
-
- /**
- *
- * @param ssid
- * @param password
- */
- async connectToWifi(credentials: NetworkCredentials) {
- if (credentials.ipOverride) {
- msg("Testing stored IP address", "warning")
- const validIP = await this.testIP(credentials.ipOverride);
- msg(
- validIP ? "Validated stored IP address" : "Stored IP address could not be reached",
- validIP ? "success" : "warning"
- )
- if (validIP) return this.setIP(credentials.ipOverride);
- }
-
- msg("Asking doodlebot for it's IP", "warning");
-
- let ip = await this.getIPAddress();
-
- if (ip) {
- if (ip === localIp) {
- msg("Doodlebot IP is local, not valid", "warning");
- }
- else {
- msg("Testing Doodlebot's reported IP address", "warning");
- const validIP = await this.testIP(ip);
- msg(
- validIP ? "Validated Doodlebot's IP address" : "Doodlebot's IP address could not be reached",
- validIP ? "success" : "warning"
- )
- if (validIP) return this.setIP(ip);
- }
- }
- else {
- msg("Could not retrieve IP address from doodlebot", "error")
- }
-
- // return new Promise(async (resolve) => {
- // const self = this;
- // const { device } = this;
-
- // const reconnectToBluetooth = async () => {
- // this.requestBluetooth(async (ble) => {
- // msg("Reconnected to doodlebot", "success");
- // const { robot, services } = await Doodlebot.getBLE(ble);
- // self.attachToBLE(robot, services);
- // device.removeEventListener("gattserverdisconnected", reconnectToBluetooth);
- // msg("Waiting to issue connect command", "warning");
- // await new Promise((resolve) => setTimeout(resolve, 5000));
- // msg("Testing doodlebot's IP after reconnect", "warning");
- // const ip = await self.getIPAddress();
- // msg(
- // ip === localIp ? "Doodlebot's IP is local, not valid" : "Doodlebot's IP is valid",
- // ip === localIp ? "warning" : "success"
- // )
- // resolve(this.setIP(ip));
- // });
- // }
-
- // device.addEventListener("gattserverdisconnected", reconnectToBluetooth);
-
- // msg("Attempting to connect to wifi", "warning");
-
- // await this.sendBLECommand(command.wifi, credentials.ssid, credentials.password);
- // });
- }
-
/**
*
* @param credentials
*/
- async connectToWebsocket(ip: string) {
- this.websocket = makeWebsocket(ip, '/api/v1/command');
+ async connectToWebsocket() {
+ const tld = await this.topLevelDomain.promise;
+ this.websocket = makeWebsocket(tld, '/api/v1/command');
await this.untilFinishedPending("websocket", new Promise((resolve) => {
const resolveAndRemove = () => {
console.log("Connected to websocket");
@@ -995,60 +753,16 @@ export default class Doodlebot {
}));
}
- async connectionWorkflow(credentials: NetworkCredentials) {
- // i commented out the next line as a hack to get this working
- // quickly on my machine, it probably just needs to be uncommented
- // but I can't test this at the moment -jon
- //await this.connectToWifi(credentials);
- const urlParams = new URLSearchParams(window.location.search); // Hack for now -jon
- let ip = urlParams.get("ip");
- this.setIP(ip);
-
- await this.connectToWebsocket(this.connection.ip);
- this.detector = new LineDetector(this.connection.ip);
- //await this.connectToImageWebSocket(this.connection.ip);
+ async connectionWorkflow() {
+ const tld = await this.topLevelDomain.promise;
+ if (this.websocket) this.websocket.close();
+ await this.connectToWebsocket();
+ this.detector = new LineDetector(tld);
}
getImageStream() {
return this.canvasWebrtc;
}
-
-
-
- // async getImageStream() {
- // if (this.pending["websocket"]) await this.pending["websocket"];
- // if (!this.connection.ip) return;
- // const image = document.createElement("img");
- // image.src = `https://${this.connection.ip}/api/v1/videopush/${endpoint.video}`;
- // image.crossOrigin = "anonymous";
- // await new Promise((resolve) => image.addEventListener("load", resolve));
- // return image;
- // }
-
- async connectToImageWebSocket(ip: string) {
- // Create a WebSocket connection
- this.websocket = new WebSocket(`wss://${ip}:${port.camera}`);
-
- // Return a promise that resolves when the WebSocket is connected
- await this.untilFinishedPending("image", new Promise((resolve, reject) => {
- const onOpen = () => {
- console.log("Connected to WebSocket for image stream");
- this.websocket.removeEventListener("open", onOpen);
- resolve();
- };
-
- const onError = (err: Event) => {
- console.error("WebSocket error: ", err);
- reject(err);
- };
-
- this.websocket.addEventListener("open", onOpen);
- this.websocket.addEventListener("error", onError);
-
- // Handle each message (which could be an image frame)
- this.websocket.addEventListener("message", (event) => this.onWebSocketImageMessage(event));
- }));
- }
// Handle incoming image data from WebSocket
onWebSocketImageMessage(event: MessageEvent) {
@@ -1114,11 +828,11 @@ export default class Doodlebot {
deepEqual = (a, b) => {
if (a === b) return true;
if (Array.isArray(a) && Array.isArray(b)) {
- return a.length === b.length && a.every((val, i) => this.deepEqual(val, b[i]));
+ return a.length === b.length && a.every((val, i) => this.deepEqual(val, b[i]));
}
if (typeof a === 'object' && typeof b === 'object') {
- const keysA = Object.keys(a), keysB = Object.keys(b);
- return keysA.length === keysB.length && keysA.every(key => this.deepEqual(a[key], b[key]));
+ const keysA = Object.keys(a), keysB = Object.keys(b);
+ return keysA.length === keysB.length && keysA.every(key => this.deepEqual(a[key], b[key]));
}
return false;
};
@@ -1131,9 +845,11 @@ export default class Doodlebot {
let t = { aT: 0.3 };
let lastTime: number;
- console.log(this.connection.ip);
+ const tld = await this.topLevelDomain.promise;
+
+ console.log(tld);
console.log(this.detector);
- this.detector = new LineDetector(this.connection.ip);
+ this.detector = new LineDetector(tld);
await this.detector.initialize(this);
let prevLine = [];
let add = 0;
@@ -1153,7 +869,7 @@ export default class Doodlebot {
this.lineCounter += 1;
let newMotorCommands;
-
+
if (first) {
({ motorCommands: newMotorCommands, bezierPoints: this.bezierPoints, line: this.line } = followLine(
lineData,
@@ -1171,13 +887,13 @@ export default class Doodlebot {
lineData,
prevLine,
this.motorCommands,
- [prevInterval/2],
+ [prevInterval / 2],
[t.aT],
add,
false
));
}
-
+
lastTime = Date.now();
// Debugging statement
@@ -1191,7 +907,7 @@ export default class Doodlebot {
newMotorCommands[0].angle = -10;
}
-
+
if (this.iterationNumber % iterations == 0) {
newMotorCommands[0].angle = this.limitArcLength(newMotorCommands[0].angle, newMotorCommands[0].radius, 2);
// newMotorCommands[0].angle = this.increaseArcLength(newMotorCommands[0].angle, newMotorCommands[0].radius, );
@@ -1206,14 +922,14 @@ export default class Doodlebot {
t = calculateArcTime(0, 0, newMotorCommands[0].radius, newMotorCommands[0].angle);
}
}
-
+
this.motorCommands = newMotorCommands;
for (const command of this.motorCommands) {
let { radius, angle } = command;
if ((lineData.length == 0 || !this.deepEqual(lineData, prevLine))) {
if (command.distance > 0) {
- this.sendWebsocketCommand("m", Math.round(12335.6*command.distance), Math.round(12335.6*command.distance), 500, 500);
+ this.sendWebsocketCommand("m", Math.round(12335.6 * command.distance), Math.round(12335.6 * command.distance), 500, 500);
} else {
this.sendBLECommand("t", radius, angle);
}
@@ -1221,7 +937,7 @@ export default class Doodlebot {
if (this.deepEqual(lineData, prevLine) && lineData.length > 0) {
console.log("LAG");
}
-
+
}
if (prevLine.length < 100 && lineData.length < 100) {
add = add + 1;
@@ -1229,9 +945,9 @@ export default class Doodlebot {
add = 0;
}
}
-
+
await new Promise((resolve) => setTimeout(resolve, waitTime));
- prevInterval = waitTime/1000;
+ prevInterval = waitTime / 1000;
first = false;
prevLine = lineData;
} catch (error) {
@@ -1251,7 +967,7 @@ export default class Doodlebot {
}
angle = Math.abs(angle);
const maxAngle = (maxArcLength * 180) / ((radius + 2.93) * Math.PI);
-
+
const returnAngle = Math.min(angle, maxAngle);
// Return the limited angle
return negative ? returnAngle * -1 : returnAngle;
@@ -1265,17 +981,17 @@ export default class Doodlebot {
}
angle = Math.abs(angle);
const maxAngle = (maxArcLength * 180) / ((radius + 2.93) * Math.PI);
-
+
// Return the limited angle
- return negative ? maxAngle*-1 : maxAngle;
+ return negative ? maxAngle * -1 : maxAngle;
}
- private setupAudioStream() {
- if (!this.connection.ip) return false;
-
+ private async setupAudioStream() {
if (this.audioSocket) return true;
- const socket = new WebSocket(`wss://${this.connection.ip}:${port.audio}`);
+ const tld = await this.topLevelDomain.promise;
+
+ const socket = new WebSocket(`wss://${tld}:${port.audio}`);
const self = this;
socket.onopen = function (event) {
@@ -1353,12 +1069,12 @@ export default class Doodlebot {
console.warn("WebSocket is already sending audio data.");
return;
}
-
+
this.isSendingAudio = true;
let CHUNK_SIZE = 1024;
- let ip = this.connection.ip;
- const ws = makeWebsocket(ip, '/api/v1/speaker');
+ const tld = await this.topLevelDomain.promise;
+ const ws = makeWebsocket(tld, '/api/v1/speaker');
ws.onopen = () => {
console.log('WebSocket connection opened');
let { sampleWidth, channels, rate } = this.parseWavHeader(uint8Array);
@@ -1408,10 +1124,10 @@ export default class Doodlebot {
const startTime = Date.now();
const callbacks = this.audioCallbacks;
-
+
return new Promise<{ context: AudioContext, buffer: AudioBuffer }>((resolve) => {
- const accumulate = (chunk: Float32Array) => {
+ const accumulate = (chunk: Float32Array) => {
// Check if we've exceeded our time limit
if (Date.now() - startTime >= numSeconds * 1000) {
callbacks.delete(accumulate);
@@ -1462,30 +1178,28 @@ export default class Doodlebot {
async moveEyes(direction1: string, direction2: string) {
const dirMap: Record = {
- center: "C",
- left: "<",
- right: ">",
- up: "^",
- down: "v",
+ center: "C",
+ left: "<",
+ right: ">",
+ up: "^",
+ down: "v",
};
- //const websocket2 = new WebSocket(`ws://${this.connection.ip}:${8766}`);
-
const from = dirMap[direction1];
const to = dirMap[direction2];
-
+
if (!from || !to || (from != "C" && to != "C")) {
- throw new Error(`Invalid direction: ${direction1}, ${direction2}`);
+ throw new Error(`Invalid direction: ${direction1}, ${direction2}`);
}
-
+
const movement = `${from}${to}`;
-
+
await this.sendWebsocketCommand(command.display, movement);
- }
+ }
- async displayFile(file: string) {
+ async displayFile(file: string) {
await this.sendWebsocketCommand(command.display, file);
- }
+ }
/**
* NOTE: Consider making private
@@ -1493,8 +1207,9 @@ export default class Doodlebot {
* @param args
* @returns
*/
- sendBLECommand(command: Command, ...args: (string | number)[]) {
- return this.ble.send(this.formCommand(command, ...args));
+ async sendBLECommand(command: Command, ...args: (string | number)[]) {
+ const ble = await this.ble.promise;
+ return ble.send(this.formCommand(command, ...args));
}
/**
diff --git a/extensions/src/doodlebot/LineDetection.ts b/extensions/src/doodlebot/LineDetection.ts
index c03a25cdf..7175f42fe 100644
--- a/extensions/src/doodlebot/LineDetection.ts
+++ b/extensions/src/doodlebot/LineDetection.ts
@@ -24,7 +24,7 @@ export class LineDetector {
private readonly MIN_PROCESS_INTERVAL = 100;
private imageStream: HTMLImageElement | null = null;
- constructor(private raspberryPiIp: string, imageStream: HTMLImageElement, width = 640, height = 480) {
+ constructor(private raspberryPiIp: string, imageStream: HTMLImageElement = null, width = 640, height = 480) {
debug.info('Initializing LineDetector', { raspberryPiIp, width, height });
this.width = width;
@@ -110,56 +110,56 @@ export class LineDetector {
try {
- try {
+ try {
- debug.info('Clearing canvas and drawing new image');
- this.ctx.clearRect(0, 0, this.width, this.height);
- this.ctx.drawImage(this.imageStream!, 0, 0, this.width, this.height);
+ debug.info('Clearing canvas and drawing new image');
+ this.ctx.clearRect(0, 0, this.width, this.height);
+ this.ctx.drawImage(this.imageStream!, 0, 0, this.width, this.height);
- debug.info('Getting image data from canvas');
- const imageData = this.ctx.getImageData(0, 0, this.width, this.height);
+ debug.info('Getting image data from canvas');
+ const imageData = this.ctx.getImageData(0, 0, this.width, this.height);
- debug.info('Processing image data');
- const lineCoordinates = this.processImageData(imageData);
+ debug.info('Processing image data');
+ const lineCoordinates = this.processImageData(imageData);
- if (lineCoordinates.length > 0) {
- debug.info('Line detected successfully', {
- points: lineCoordinates.length,
- firstPoint: lineCoordinates[0],
- lastPoint: lineCoordinates[lineCoordinates.length - 1]
- });
+ if (lineCoordinates.length > 0) {
+ debug.info('Line detected successfully', {
+ points: lineCoordinates.length,
+ firstPoint: lineCoordinates[0],
+ lastPoint: lineCoordinates[lineCoordinates.length - 1]
+ });
- this.lastDetectedLine = lineCoordinates;
+ this.lastDetectedLine = lineCoordinates;
- if (this.collectLine) {
- this.allCoordinates.push(lineCoordinates);
- this.frameCount++;
- this.collectLine = false;
- debug.info('Line collected, frame count:', this.frameCount);
- }
-
- this.lastProcessTime = now;
- this.isProcessing = false;
- debug.timeEnd('detectLine');
- return lineCoordinates;
- } else {
- this.lastDetectedLine = lineCoordinates;
+ if (this.collectLine) {
+ this.allCoordinates.push(lineCoordinates);
+ this.frameCount++;
+ this.collectLine = false;
+ debug.info('Line collected, frame count:', this.frameCount);
}
- debug.warn('No line detected in this attempt');
- attempt++;
- const backoffTime = 100 * Math.pow(2, attempt);
- debug.info(`Waiting ${backoffTime}ms before next attempt`);
+ this.lastProcessTime = now;
+ this.isProcessing = false;
+ debug.timeEnd('detectLine');
+ return lineCoordinates;
+ } else {
+ this.lastDetectedLine = lineCoordinates;
+ }
- } catch (error) {
- debug.error(`Processing attempt ${attempt + 1} failed:`, error);
- attempt++;
+ debug.warn('No line detected in this attempt');
+ attempt++;
+ const backoffTime = 100 * Math.pow(2, attempt);
+ debug.info(`Waiting ${backoffTime}ms before next attempt`);
-
+ } catch (error) {
+ debug.error(`Processing attempt ${attempt + 1} failed:`, error);
+ attempt++;
- const backoffTime = 100 * Math.pow(2, attempt);
- debug.info(`Waiting ${backoffTime}ms before retry`);
- }
+
+
+ const backoffTime = 100 * Math.pow(2, attempt);
+ debug.info(`Waiting ${backoffTime}ms before retry`);
+ }
} catch (error) {
debug.error('Error getting image stream:', error);
diff --git a/extensions/src/doodlebot/ble.ts b/extensions/src/doodlebot/ble.ts
new file mode 100644
index 000000000..198fe67a3
--- /dev/null
+++ b/extensions/src/doodlebot/ble.ts
@@ -0,0 +1,24 @@
+import UartService from "./communication/UartService";
+
+export type BLEDeviceWithUartService = { device: BluetoothDevice, service: UartService };
+
+export const getBLEDeviceWithUartService = async (
+ bluetooth: Bluetooth, ...filters: BluetoothLEScanFilter[]
+): Promise => {
+ const device = await bluetooth.requestDevice({
+ filters: [
+ ...(filters ?? []),
+ {
+ services: [UartService.uuid]
+ },
+ ],
+ });
+ if (!device) return { error: "No device selected" };
+ if (!device.gatt) return { error: "No GATT server found" };
+ if (!device.gatt.connected) await device.gatt.connect();
+ const services = await device.gatt.getPrimaryServices();
+ const found = services.find((service) => service.uuid === UartService.uuid);
+ if (!found) return { error: "UART service not found" };
+ const service = await UartService.create(found);
+ return { device, service };
+}
diff --git a/extensions/src/doodlebot/index.ts b/extensions/src/doodlebot/index.ts
index 67035e8f9..35aee604e 100644
--- a/extensions/src/doodlebot/index.ts
+++ b/extensions/src/doodlebot/index.ts
@@ -1,19 +1,14 @@
import { Environment, ExtensionMenuDisplayDetails, extension, block, buttonBlock, BlockUtilityWithID, scratch } from "$common";
import { DisplayKey, displayKeys, command, type Command, SensorKey, sensorKeys, units, keyBySensor, sensor } from "./enums";
-import Doodlebot, { NetworkCredentials } from "./Doodlebot";
-import FileArgument from './FileArgument.svelte';
-import { splitArgsString } from "./utils";
+import Doodlebot from "./Doodlebot";
import EventEmitter from "events";
-import { categoryByGesture, classes, emojiByGesture, gestureDetection, gestureMenuItems, gestures, objectDetection } from "./detection";
+import { categoryByGesture } from "./detection";
//import { createLineDetector } from "./LineDetection";
-import { line0, line1, line2, line3, line4, line5, line6, line7, line8 } from './Points';
-import { followLine } from "./LineFollowing";
-import { createLineDetector } from "./LineDetection";
import tmPose from '@teachablemachine/pose';
-import { calculateArcTime } from "./TimeHelper";
import tmImage from '@teachablemachine/image';
import * as speechCommands from '@tensorflow-models/speech-commands';
import JSZip from 'jszip';
+import type { BLEDeviceWithUartService } from "./ble";
const details: ExtensionMenuDisplayDetails = {
name: "Doodlebot",
@@ -49,7 +44,9 @@ export var imageFiles: string[] = [];
export var soundFiles: string[] = [];
export default class DoodlebotBlocks extends extension(details, "ui", "customArguments", "indicators", "video", "drawable") {
- doodlebot: Doodlebot;
+ doodlebot = new Doodlebot();
+ connected = false;
+
private indicator: Promise<{ close(): void; }>;
private lineDetector: (() => Promise) | null = null;
bluetoothEmitter = new EventEmitter();
@@ -88,8 +85,6 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
soundDictionary: {} | { string: string[] };
costumeDictionary: {} | { string: string[] };
- externalIp: string
-
voice_id: number;
pitch_value: number;
@@ -122,13 +117,12 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
this.pitch_value = 0;
this.soundDictionary = {};
this.costumeDictionary = {};
- this.setIndicator("disconnected");
- if (window.isSecureContext) this.openUI("Connect")
- else this.connectToDoodlebotWithExternalBLE();
+ //requestAnimationFrame(() => this.setIndicator("disconnected"));
+ this.openUI("Connect")
this._loop();
env.runtime.on("TARGETS_UPDATE", async () => {
await this.setDictionaries();
- })
+ })
await this.setDictionaries();
@@ -140,7 +134,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
// const response = await fetch(url, {
// method: "POST"
// });
-
+
// if (!response.ok) {
// const text = await response.text();
// console.error("Error setting voice/pitch:", text);
@@ -150,7 +144,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
// }
// env.runtime.on("PROJECT_RUN_START", async () => {
-
+
// })
// env.runtime.on("PROJECT_RUN_STOP", async () => {
// if (this.blocksRun > 10) {
@@ -176,7 +170,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
// }
// })
-
+
}
async setDictionaries() {
@@ -251,173 +245,23 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
});
}
- private async connectToDoodlebotWithExternalBLE() {
- // A few globals that currently must be set the same across the playground and the https frontend
- const handshakeMessage = "doodlebot";
- const disconnectMessage = "disconnected";
- const commandCompleteIdentifier = "done";
-
- const urlParams = new URLSearchParams(window.location.search); // Hack for now -jon
-
- const ip = urlParams.get("ip");
-
- if (!ip) {
- alert("No IP address provided. Please provide an IP address in the URL query string.");
- return;
- }
-
- this.externalIp = ip;
-
- const networkCredentials: NetworkCredentials = {
- ssid: "dummy", // NOTE: When using the external BLE, it is assumed a valid ip address will be provided, and thus there is no need for wifi credentials
- password: "dummy", // NOTE: When using the external BLE, it is assumed a valid ip address will be provided, and thus there is no need for wifi credentials
- ipOverride: ip
- }
-
- type ExternalPageDetails = { source: MessageEventSource, targetOrigin: string }
-
- const { source, targetOrigin } = await new Promise((resolve) => {
- const onInitialMessage = ({ data, source, origin }: MessageEvent) => {
- if (typeof data !== "string" || data !== handshakeMessage) return;
- window.removeEventListener("message", onInitialMessage);
- console.log("posting ready");
- source.postMessage("ready", { targetOrigin: origin })
- resolve({ source, targetOrigin: origin });
- }
- window.addEventListener("message", onInitialMessage);
- });
-
- console.log("source", source);
- console.log("target origin", targetOrigin);
-
- const doodlebot = new Doodlebot(
- {
- send: (text) => new Promise(resolve => {
- const onMessageReturn = ({ data, origin }: MessageEvent) => {
- if (origin !== targetOrigin || !data.includes(text) || !data.includes(commandCompleteIdentifier)) {
- console.log("error -- source");
- return;
- }
- window.removeEventListener("message", onMessageReturn);
- resolve();
- }
- window.addEventListener("message", onMessageReturn);
- console.log("posting message");
- source.postMessage(text, { targetOrigin });
- }),
-
- onReceive: (callback) => {
- window.addEventListener('message', ({ data, origin }) => {
- console.log("RECEIVED", data);
- if (origin !== targetOrigin || data === disconnectMessage || data.includes(commandCompleteIdentifier)) {
- console.log("error 2 -- source", data);
- return;
- }
- callback(new CustomEvent("ble", { detail: data }));
- });
- },
-
- onDisconnect: () => {
- window.addEventListener("message", ({ data, origin }) => {
- if (origin !== targetOrigin || data !== disconnectMessage) return;
- this.setIndicator("disconnected");
- alert("Disconnected from robot"); // Decide how to handle (maybe direct user to close window and go back to https)
- });
- },
- },
- () => alert("requestBluetooth called"), // placeholder
- networkCredentials,
- () => alert("save IP called"), // placeholder,
- async (description) => {
- // Send the fetch request to the source
- console.log("INSIDE FETCH 2");
-
- return new Promise((resolve, reject) => {
- console.log("INSIDE PROMISE 2");
- const fetchReturn = (event: MessageEvent) => {
- console.log("inside return");
- if (event.origin !== targetOrigin) {
- console.log("ERROR", event.origin, targetOrigin);
- return;
- }
- if (!event.data.startsWith("fetchResponse---")) {
- console.log("ERROR", event.data);
- return;
- }
- const urlReturned = event.data.split("---")[1];
- if ("webrtc" != urlReturned) {
- console.log("URL NOT SAME");
- return;
- }
- const response = event.data.split("---")[2];
- console.log("RESPONSE", JSON.parse(response));
- window.removeEventListener('message', fetchReturn);
- resolve(JSON.parse(response));
- }
- console.log("adding return");
- window.addEventListener('message', fetchReturn);
- console.log("posting message", `fetch---webrtc---${description}`, targetOrigin);
- source.postMessage(`fetch---webrtc---${description}`, { targetOrigin });
- });
- }
- )
- doodlebot.fetch = async (url: string, type: string, options?: string) => {
- // Send the fetch request to the source
- return new Promise((resolve, reject) => {
- const fetchReturn = (event: MessageEvent) => {
- if (event.origin !== targetOrigin) {
- console.log("ERROR", event.origin, targetOrigin);
- return;
- }
- if (!event.data.startsWith("fetchResponse---")) {
- console.log("ERROR", event.data);
- return;
- }
- const urlReturned = event.data.split("---")[1];
- if (url != urlReturned) {
- console.log("URL NOT SAME");
- return;
- }
- const response = event.data.split("---")[2];
- console.log("RESPONSE", response);
- window.removeEventListener('message', fetchReturn);
- resolve(response);
- }
- console.log("adding return");
- window.addEventListener('message', fetchReturn);
- console.log("posting message", `fetch---${type}--${url}`, targetOrigin);
- if (options) {
- source.postMessage(`fetch---${type}---${url}---${JSON.stringify(options)}`, { targetOrigin });
- } else {
- source.postMessage(`fetch---${type}---${url}`, { targetOrigin });
- }
- });
- }
- doodlebot.setIP(ip);
- this.setDoodlebot(doodlebot);
-
-
- }
-
getCurrentSounds(id): string[] {
return (this.soundDictionary && this.soundDictionary[id]) ? Object.keys(this.soundDictionary[id]) : [];
}
- async setDoodlebot(doodlebot: Doodlebot) {
- this.doodlebot = doodlebot;
- await this.setIndicator("connected");
+ async setDoodlebot(topLevelDomain: string, bluetooth: BLEDeviceWithUartService) {
+ this.doodlebot.topLevelDomain.resolve(topLevelDomain);
+ this.doodlebot.bleDevice.resolve(bluetooth);
- const urlParams = new URLSearchParams(window.location.search); // Hack for now -jon
- const ip = urlParams.get("ip");
- this.doodlebot.setIP(ip);
+ await this.setIndicator("connected");
try {
- imageFiles = await doodlebot.findImageFiles();
- soundFiles = await doodlebot.findSoundFiles();
+ imageFiles = await this.doodlebot.findImageFiles();
+ soundFiles = await this.doodlebot.findSoundFiles();
} catch (e) {
//this.openUI("ArrayError");
}
-
+
// Wait a short moment to ensure connection is established
await new Promise(resolve => setTimeout(resolve, 1000));
@@ -433,7 +277,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
}
});
-
+
try {
if (this.SOCIAL && Math.random() < this.socialness && this.doodlebot) {
await this.doodlebot.display("happy");
@@ -445,19 +289,6 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
}
await this.doodlebot.display("happy");
-
- try {
- console.log("FETCHING");
- const ip = await this.getIPAddress();
- console.log(doodlebot.fetch);
- let tempImage = await doodlebot.fetch(`http://${ip}:8080/images`, "text");
- imageFiles = doodlebot.extractList(tempImage);
- console.log("FILES", imageFiles)
- let tempSound = await doodlebot.fetch(`http://${ip}:8080/sounds`, "text");
- soundFiles = doodlebot.extractList(tempSound);
- } catch (e) {
- //this.openUI("ArrayError");
- }
}
async setIndicator(status: "connected" | "disconnected") {
@@ -501,7 +332,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
//console.error("Failed to get image stream");
return;
}
-
+
let stageWidth = 480;
let stageHeight = 360;
@@ -509,22 +340,22 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
resizedCanvas.width = stageWidth;
resizedCanvas.height = stageHeight;
const resizedCtx = resizedCanvas.getContext('2d');
-
+
const drawable = this.createDrawable(resizedCanvas); // draw from resized version
drawable.setVisible(true);
-
+
const update = () => {
const latest = this.doodlebot?.getImageStream();
if (!latest) return;
-
+
// Draw the current stream into the resized canvas
resizedCtx.clearRect(0, 0, stageWidth, stageHeight);
resizedCtx.drawImage(latest, 0, 0, stageWidth, stageHeight);
drawable.update(resizedCanvas);
-
+
requestAnimationFrame(update);
};
-
+
requestAnimationFrame(update);
return drawable;
}
@@ -545,7 +376,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
// async setSocialness(value: number) {
// // Ensure value is between 0 and 1
// this.socialness = Math.max(0, Math.min(1, value));
-
+
// if (this.SOCIAL && Math.random() < this.socialness) {
// await this.doodlebot?.display("happy");
// await this.speakText(`I'll be ${Math.round(this.socialness * 100)}% social from now on!`);
@@ -574,15 +405,15 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
type: "command",
text: (voice, pitch) => `set voice to ${voice} and pitch to ${pitch}`,
args: [
- { type: "number", defaultValue: 1, name: "voice" },
- { type: "number", defaultValue: 0, name: "pitch" }
+ { type: "number", defaultValue: 1 },
+ { type: "number", defaultValue: 0 }
]
})
async setVoiceAndPitch(voice: number, pitch: number) {
this.voice_id = voice;
this.pitch_value = pitch;
}
-
+
// @block({
// type: "command",
@@ -592,14 +423,14 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
// async repeatAfterMe(seconds: number) {
// // Record the audio
// const { context, buffer } = await this.doodlebot?.recordAudio(seconds);
-
+
// // Convert to WAV format
// const wavBlob = await this.saveAudioBufferToWav(buffer);
// const arrayBuffer = await wavBlob.arrayBuffer();
-
+
// // Send the audio data directly to the Doodlebot for playback
// await this.doodlebot.sendAudioData(new Uint8Array(arrayBuffer));
-
+
// // Wait until playback is complete (approximately buffer duration)
// const playbackDuration = buffer.duration * 1000; // convert to milliseconds
// await new Promise(resolve => setTimeout(resolve, playbackDuration));
@@ -764,7 +595,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
type: "reporter",
text: (axis: string, sensor: SensorKey) => `get ${axis} of ${sensor} sensor`,
args: [
- {type: 'string', options: ["x", "y", "z"], defaultValue: 'x'},
+ { type: 'string', options: ["x", "y", "z"], defaultValue: 'x' },
{ type: "string", options: ["gyroscope", "accelerometer"], defaultValue: "gyroscope" }
]
})
@@ -776,7 +607,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
return `${JSON.stringify(reading[axis])} ${units[sensor]}`;
}
-
+
@(scratch.hat`
when ${{ type: "string", options: ["battery", "temperature", "humidity", "pressure", "distance", "altimeter"], defaultValue: "battery" }}
@@ -820,7 +651,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
})
whenBumperPressed(bumber: typeof bumperOptions[number], condition: "release" | "pressed") {
const isPressed = this.doodlebot?.getSensorReadingSync("bumper");
-
+
const isPressedCondition = condition === "pressed";
if (!isPressed) {
return false;
@@ -931,7 +762,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
(self.costumeDictionary && self.costumeDictionary[self.runtime._editingTarget.id]) ? Object.keys(self.costumeDictionary[self.runtime._editingTarget.id]) : [] as any[]
).filter((item: string) => item != "costume9999.png")
}, defaultValue: "happy"
- }, {type: "string", defaultValue: 1}]
+ }, { type: "string", defaultValue: 1 }]
}))
async setDisplayForSeconds(display: DisplayKey | string, seconds: number) {
const lastDisplayedKey = this.doodlebot.getLastDisplayedKey();
@@ -941,28 +772,20 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
await this.uploadFile("image", this.costumeDictionary[this.runtime._editingTarget.id][display]);
await this.setArrays();
await this.doodlebot.displayFile("costume9999.png");
- await new Promise(resolve => setTimeout(resolve, seconds*1000));
+ await new Promise(resolve => setTimeout(resolve, seconds * 1000));
} else if (imageFiles.includes(display)) {
await this.doodlebot?.displayFile(display);
- await new Promise(resolve => setTimeout(resolve, seconds*1000));
+ await new Promise(resolve => setTimeout(resolve, seconds * 1000));
} else {
await this.doodlebot?.display(display as DisplayKey);
- await new Promise(resolve => setTimeout(resolve, seconds*1000));
+ await new Promise(resolve => setTimeout(resolve, seconds * 1000));
}
if (lastDisplayedType == "text") {
await this.doodlebot.displayText(lastDisplayedKey);
} else {
await this.doodlebot.display(lastDisplayedKey);
}
-
- }
- async getIPAddress() {
- if (window.isSecureContext) {
- return this.doodlebot?.getStoredIPAddress();
- } else {
- return this.externalIp;
- }
}
// @block({
@@ -1077,7 +900,7 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
// await new Promise((resolve) => setTimeout(resolve, audioDuration * 1000));
// }
-
+
async setArrays() {
imageFiles = await this.doodlebot.findImageFiles();
soundFiles = await this.doodlebot.findSoundFiles();
@@ -1086,13 +909,13 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
async uploadFile(type: string, blobURL: string) {
console.log("BEFORE IP");
- const ip = await this.getIPAddress();
+ const tld = await this.doodlebot.topLevelDomain.promise;
console.log("GOT IP");
let uploadEndpoint;
if (type == "sound") {
- uploadEndpoint = "https://" + ip + "/api/v1/upload/sounds_upload";
+ uploadEndpoint = "https://" + tld + "/api/v1/upload/sounds_upload";
} else {
- uploadEndpoint = "https://" + ip + "/api/v1/upload/img_upload";
+ uploadEndpoint = "https://" + tld + "/api/v1/upload/img_upload";
}
try {
@@ -1106,50 +929,38 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
throw new Error(`Failed to fetch Blob from URL: ${blobURL}`);
}
const blob = await response1.blob();
- if (window.isSecureContext) {
- console.log("BEFORE BLOB 2");
-
- console.log("AFTER BLOB 2");
- // Convert Blob to File
- const file = new File([blob], components[0], { type: blob.type });
- const formData = new FormData();
- formData.append("file", file);
-
- console.log("file");
- console.log(file);
- console.log("BEFORE FETCH");
- const response2 = await fetch(uploadEndpoint, {
- method: "POST",
- body: formData,
- });
- console.log("AFTER FETCH");
- console.log(response2);
-
- if (!response2.ok) {
- throw new Error(`Failed to upload file: ${response2.statusText}`);
- }
+ console.log("BEFORE BLOB 2");
+
+ console.log("AFTER BLOB 2");
+ // Convert Blob to File
+ const file = new File([blob], components[0], { type: blob.type });
+ const formData = new FormData();
+ formData.append("file", file);
+
+ console.log("file");
+ console.log(file);
+ console.log("BEFORE FETCH");
+ const response2 = await fetch(uploadEndpoint, {
+ method: "POST",
+ body: formData,
+ });
+ console.log("AFTER FETCH");
+ console.log(response2);
- console.log("File uploaded successfully");
- this.setArrays();
- } else {
- const base64 = await this.blobToBase64(blob);
- const payload = {
- filename: components[0],
- content: base64,
- mimeType: blob.type,
- };
- const response2 = await this.doodlebot.fetch(uploadEndpoint, "file_upload", payload);
+ if (!response2.ok) {
+ throw new Error(`Failed to upload file: ${response2.statusText}`);
}
-
+
+ console.log("File uploaded successfully");
+ this.setArrays();
} catch (error) {
console.error("Error:", error);
}
}
async callSinglePredict() {
- const ip = await this.getIPAddress();
- return await this.doodlebot.callSinglePredict(this.getIPAddress());
-
+ return await this.doodlebot.callSinglePredict();
+
}
@block({
@@ -1161,19 +972,17 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
]
})
async getSinglePredict2s(location: string, type: "face" | "object") {
- const ip = await this.getIPAddress();
- const reading = await this.doodlebot.getFacePrediction(ip, type);
+ const reading = await this.doodlebot.getFacePrediction(type);
return this.doodlebot.getReadingLocation(location, type == "object" ? "apple" : type, reading);
}
@block({
type: "Boolean",
text: (type) => `is ${type} detected`,
- args: [{ type: "string", options: ["face", "apple", "orange"], defaultValue: "face" }]
+ arg: { type: "string", options: ["face", "apple", "orange"], defaultValue: "face" }
})
async isFaceDetected(type: string) {
- const ip = await this.getIPAddress();
- const reading = await this.doodlebot.getFacePrediction(ip, "face");
+ const reading = await this.doodlebot.getFacePrediction("face");
const x = this.doodlebot.getReadingLocation("x", type, reading);
const y = this.doodlebot.getReadingLocation("y", type, reading);
if (x == -1 && y == -1) {
@@ -1212,17 +1021,10 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
await this.doodlebot?.setVolume(volume)
}
-
-
- // @block({
- // type: "reporter",
- // text: "get IP address"
- // })
- async getIP() {
- return this.doodlebot?.getIPAddress();
- }
+
+
// @block({
// type: "command",
@@ -1314,67 +1116,67 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
writeString(view: DataView, offset: number, text: string) {
for (let i = 0; i < text.length; i++) {
- view.setUint8(offset + i, text.charCodeAt(i));
+ view.setUint8(offset + i, text.charCodeAt(i));
}
}
async saveAudioBufferToWav(buffer) {
function createWavHeader(buffer) {
- const numChannels = buffer.numberOfChannels;
- const sampleRate = buffer.sampleRate / 4;
- const bitsPerSample = 16; // 16-bit PCM
- const blockAlign = (numChannels * bitsPerSample) / 8;
- const byteRate = sampleRate * blockAlign;
- const dataLength = buffer.length * numChannels * 2; // 16-bit PCM = 2 bytes per sample
- const header = new ArrayBuffer(44);
- const view = new DataView(header);
- // "RIFF" chunk descriptor
- writeString(view, 0, "RIFF");
- view.setUint32(4, 36 + dataLength, true); // File size - 8 bytes
- writeString(view, 8, "WAVE");
- // "fmt " sub-chunk
- writeString(view, 12, "fmt ");
- view.setUint32(16, 16, true); // Sub-chunk size (16 for PCM)
- view.setUint16(20, 1, true); // Audio format (1 = PCM)
- view.setUint16(22, numChannels, true); // Number of channels
- view.setUint32(24, sampleRate, true); // Sample rate
- view.setUint32(28, byteRate, true); // Byte rate
- view.setUint16(32, blockAlign, true); // Block align
- view.setUint16(34, bitsPerSample, true); // Bits per sample
- // "data" sub-chunk
- writeString(view, 36, "data");
- view.setUint32(40, dataLength, true); // Data length
- console.log("WAV Header:", new Uint8Array(header));
- return header;
+ const numChannels = buffer.numberOfChannels;
+ const sampleRate = buffer.sampleRate / 4;
+ const bitsPerSample = 16; // 16-bit PCM
+ const blockAlign = (numChannels * bitsPerSample) / 8;
+ const byteRate = sampleRate * blockAlign;
+ const dataLength = buffer.length * numChannels * 2; // 16-bit PCM = 2 bytes per sample
+ const header = new ArrayBuffer(44);
+ const view = new DataView(header);
+ // "RIFF" chunk descriptor
+ writeString(view, 0, "RIFF");
+ view.setUint32(4, 36 + dataLength, true); // File size - 8 bytes
+ writeString(view, 8, "WAVE");
+ // "fmt " sub-chunk
+ writeString(view, 12, "fmt ");
+ view.setUint32(16, 16, true); // Sub-chunk size (16 for PCM)
+ view.setUint16(20, 1, true); // Audio format (1 = PCM)
+ view.setUint16(22, numChannels, true); // Number of channels
+ view.setUint32(24, sampleRate, true); // Sample rate
+ view.setUint32(28, byteRate, true); // Byte rate
+ view.setUint16(32, blockAlign, true); // Block align
+ view.setUint16(34, bitsPerSample, true); // Bits per sample
+ // "data" sub-chunk
+ writeString(view, 36, "data");
+ view.setUint32(40, dataLength, true); // Data length
+ console.log("WAV Header:", new Uint8Array(header));
+ return header;
}
function writeString(view, offset, string) {
- for (let i = 0; i < string.length; i++) {
- view.setUint8(offset + i, string.charCodeAt(i));
- }
+ for (let i = 0; i < string.length; i++) {
+ view.setUint8(offset + i, string.charCodeAt(i));
+ }
}
function interleave(buffer) {
- const numChannels = buffer.numberOfChannels;
- const length = buffer.length * numChannels;
- const result = new Float32Array(length);
- const channelData = [];
- for (let i = 0; i < numChannels; i++) {
- channelData.push(buffer.getChannelData(i));
- }
- let index = 0;
- for (let i = 0; i < buffer.length; i++) {
- for (let j = 0; j < numChannels; j++) {
- result[index++] = channelData[j][i];
- }
+ const numChannels = buffer.numberOfChannels;
+ const length = buffer.length * numChannels;
+ const result = new Float32Array(length);
+ const channelData = [];
+ for (let i = 0; i < numChannels; i++) {
+ channelData.push(buffer.getChannelData(i));
+ }
+ let index = 0;
+ for (let i = 0; i < buffer.length; i++) {
+ for (let j = 0; j < numChannels; j++) {
+ result[index++] = channelData[j][i];
}
- console.log("Interleaved data:", result);
- return result;
+ }
+ console.log("Interleaved data:", result);
+ return result;
}
function floatTo16BitPCM(output, offset, input) {
- for (let i = 0; i < input.length; i++, offset += 2) {
- let s = Math.max(-1, Math.min(1, input[i])); // Clamp to [-1, 1]
- s = s < 0 ? s * 0x8000 : s * 0x7FFF; // Convert to 16-bit PCM
- output.setInt16(offset, s, true); // Little-endian
- }
+ for (let i = 0; i < input.length; i++, offset += 2) {
+ let s = Math.max(-1, Math.min(1, input[i])); // Clamp to [-1, 1]
+ s = s < 0 ? s * 0x8000 : s * 0x7FFF; // Convert to 16-bit PCM
+ output.setInt16(offset, s, true); // Little-endian
+ }
}
const header = createWavHeader(buffer);
const interleaved = interleave(buffer);
@@ -1389,68 +1191,68 @@ export default class DoodlebotBlocks extends extension(details, "ui", "customArg
console.log("Expected data length:", header.byteLength + interleaved.length * 2);
// Return a Blob
return new Blob([wavBuffer], { type: "audio/wav" });
-}
+ }
-generateWAV(interleaved: Float32Array, sampleRate: number): Uint8Array {
- const numChannels = 1; // Mono
- const bitsPerSample = 16;
- const byteRate = sampleRate * numChannels * (bitsPerSample / 8);
- const blockAlign = numChannels * (bitsPerSample / 8);
- const dataLength = interleaved.length * (bitsPerSample / 8);
- const bufferLength = 44 + dataLength;
- const buffer = new ArrayBuffer(bufferLength);
- const view = new DataView(buffer);
- // RIFF header
- this.writeString(view, 0, "RIFF");
- view.setUint32(4, bufferLength - 8, true); // File size
- this.writeString(view, 8, "WAVE");
- // fmt subchunk
- this.writeString(view, 12, "fmt ");
- view.setUint32(16, 16, true); // Subchunk size
- view.setUint16(20, 1, true); // PCM format
- view.setUint16(22, numChannels, true); // Channels
- view.setUint32(24, sampleRate, true); // Sample rate
- view.setUint32(28, byteRate, true); // Byte rate
- view.setUint16(32, blockAlign, true); // Block align
- view.setUint16(34, bitsPerSample, true); // Bits per sample
- // data subchunk
- this.writeString(view, 36, "data");
- view.setUint32(40, dataLength, true);
- // PCM data
- const offset = 44;
- for (let i = 0; i < interleaved.length; i++) {
+ generateWAV(interleaved: Float32Array, sampleRate: number): Uint8Array {
+ const numChannels = 1; // Mono
+ const bitsPerSample = 16;
+ const byteRate = sampleRate * numChannels * (bitsPerSample / 8);
+ const blockAlign = numChannels * (bitsPerSample / 8);
+ const dataLength = interleaved.length * (bitsPerSample / 8);
+ const bufferLength = 44 + dataLength;
+ const buffer = new ArrayBuffer(bufferLength);
+ const view = new DataView(buffer);
+ // RIFF header
+ this.writeString(view, 0, "RIFF");
+ view.setUint32(4, bufferLength - 8, true); // File size
+ this.writeString(view, 8, "WAVE");
+ // fmt subchunk
+ this.writeString(view, 12, "fmt ");
+ view.setUint32(16, 16, true); // Subchunk size
+ view.setUint16(20, 1, true); // PCM format
+ view.setUint16(22, numChannels, true); // Channels
+ view.setUint32(24, sampleRate, true); // Sample rate
+ view.setUint32(28, byteRate, true); // Byte rate
+ view.setUint16(32, blockAlign, true); // Block align
+ view.setUint16(34, bitsPerSample, true); // Bits per sample
+ // data subchunk
+ this.writeString(view, 36, "data");
+ view.setUint32(40, dataLength, true);
+ // PCM data
+ const offset = 44;
+ for (let i = 0; i < interleaved.length; i++) {
const sample = Math.max(-1, Math.min(1, interleaved[i]));
view.setInt16(offset + i * 2, sample < 0 ? sample * 0x8000 : sample * 0x7FFF, true);
+ }
+ return new Uint8Array(buffer);
+ }
+
+ createAndSaveWAV(interleaved, sampleRate) {
+ // Step 1: Get interleaved audio data and sample rate
+ // Step 2: Generate WAV file
+ const wavData = this.generateWAV(interleaved, sampleRate);
+ // Step 3: Save or process the WAV file
+ // Example: Create a Blob and download the file
+ const blob = new Blob([wavData], { type: "audio/wav" });
+ const url = URL.createObjectURL(blob);
+ // Create a link to download the file
+ const a = document.createElement("a");
+ a.href = url;
+ a.download = "output.wav";
+ document.body.appendChild(a);
+ a.click();
+ document.body.removeChild(a);
+ return blob;
+ }
+
+ blobToBase64(blob) {
+ return new Promise((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onloadend = () => resolve(reader.result); // "data:;base64,"
+ reader.onerror = reject;
+ reader.readAsDataURL(blob);
+ });
}
- return new Uint8Array(buffer);
-}
-
-createAndSaveWAV(interleaved, sampleRate) {
- // Step 1: Get interleaved audio data and sample rate
- // Step 2: Generate WAV file
- const wavData = this.generateWAV(interleaved, sampleRate);
- // Step 3: Save or process the WAV file
- // Example: Create a Blob and download the file
- const blob = new Blob([wavData], { type: "audio/wav" });
- const url = URL.createObjectURL(blob);
- // Create a link to download the file
- const a = document.createElement("a");
- a.href = url;
- a.download = "output.wav";
- document.body.appendChild(a);
- a.click();
- document.body.removeChild(a);
- return blob;
-}
-
-blobToBase64(blob) {
- return new Promise((resolve, reject) => {
- const reader = new FileReader();
- reader.onloadend = () => resolve(reader.result); // "data:;base64,"
- reader.onerror = reject;
- reader.readAsDataURL(blob);
- });
-}
async sendAudioFileToChatEndpoint(file, endpoint, blob, seconds) {
console.log("sending audio file");
@@ -1462,111 +1264,111 @@ blobToBase64(blob) {
//audio.play();
try {
- let response;
- let uint8array;
- // if (window.isSecureContext) {
-
- // if (endpoint == "repeat_after_me") {
- // const eventSource = new EventSource("http://doodlebot.media.mit.edu/viseme-events");
-
- // eventSource.onmessage = (event) => {
- // console.log("Received viseme event:", event.data);
- // try {
- // const data = JSON.parse(event.data);
- // const visemeId = data.visemeId;
- // const offsetMs = data.offsetMs;
-
- // // You can customize which viseme IDs should trigger a command.
- // // For now, all non-silence visemes trigger it.
- // if (visemeId !== 0) {
- // setTimeout(() => {
- // this.doodlebot.display("happy");
- // console.log("DISPLAYING");
- // }, offsetMs);
- // }
- // } catch (err) {
- // console.error("Failed to parse viseme event:", err);
- // }
- // };
-
- // eventSource.onerror = (err) => {
- // console.error("EventSource failed:", err);
- // eventSource.close();
- // };
- // }
-
- response = await fetch(url, {
- method: "POST",
- body: formData,
- });
+ let response;
+ let uint8array;
+ // if (window.isSecureContext) {
+
+ // if (endpoint == "repeat_after_me") {
+ // const eventSource = new EventSource("http://doodlebot.media.mit.edu/viseme-events");
+
+ // eventSource.onmessage = (event) => {
+ // console.log("Received viseme event:", event.data);
+ // try {
+ // const data = JSON.parse(event.data);
+ // const visemeId = data.visemeId;
+ // const offsetMs = data.offsetMs;
+
+ // // You can customize which viseme IDs should trigger a command.
+ // // For now, all non-silence visemes trigger it.
+ // if (visemeId !== 0) {
+ // setTimeout(() => {
+ // this.doodlebot.display("happy");
+ // console.log("DISPLAYING");
+ // }, offsetMs);
+ // }
+ // } catch (err) {
+ // console.error("Failed to parse viseme event:", err);
+ // }
+ // };
+
+ // eventSource.onerror = (err) => {
+ // console.error("EventSource failed:", err);
+ // eventSource.close();
+ // };
+ // }
+
+ response = await fetch(url, {
+ method: "POST",
+ body: formData,
+ });
- if (!response.ok) {
- const errorText = await response.text();
- console.log("Error response:", errorText);
- throw new Error(`HTTP error! status: ${response.status}`);
- }
+ if (!response.ok) {
+ const errorText = await response.text();
+ console.log("Error response:", errorText);
+ throw new Error(`HTTP error! status: ${response.status}`);
+ }
+
+ const textResponse = response.headers.get("text-response");
+ console.log("Text Response:", textResponse);
+
+ const blob = await response.blob();
+ const audioUrl = URL.createObjectURL(blob);
+ console.log("Audio URL:", audioUrl);
+
+ const audio = new Audio(audioUrl);
+ const array = await blob.arrayBuffer();
+ uint8array = new Uint8Array(array);
+ // } else {
+ // const base64 = await this.blobToBase64(blob);
+ // const payload = {
+ // filename: file.name,
+ // content: base64,
+ // mimeType: blob.type,
+ // };
+ // response = await this.doodlebot.fetch(endpoint, "chatgpt", payload);
+ // uint8array = new Uint8Array([...atob(response)].map(char => char.charCodeAt(0)));
+ // }
+ const interval = 50; // 0.2 seconds in milliseconds
+ const endTime = Date.now() + 1 * 1000;
+
+ this.doodlebot.sendAudioData(uint8array);
+ while (Date.now() < endTime) {
+ await this.doodlebot.sendWebsocketCommand("d,O");
+ await new Promise((res) => setTimeout(res, interval));
+ await this.doodlebot.sendWebsocketCommand("d,N");
+ await new Promise((res) => setTimeout(res, interval));
+ }
- const textResponse = response.headers.get("text-response");
- console.log("Text Response:", textResponse);
-
- const blob = await response.blob();
- const audioUrl = URL.createObjectURL(blob);
- console.log("Audio URL:", audioUrl);
-
- const audio = new Audio(audioUrl);
- const array = await blob.arrayBuffer();
- uint8array = new Uint8Array(array);
- // } else {
- // const base64 = await this.blobToBase64(blob);
- // const payload = {
- // filename: file.name,
- // content: base64,
- // mimeType: blob.type,
- // };
- // response = await this.doodlebot.fetch(endpoint, "chatgpt", payload);
- // uint8array = new Uint8Array([...atob(response)].map(char => char.charCodeAt(0)));
- // }
- const interval = 50; // 0.2 seconds in milliseconds
- const endTime = Date.now() + 1 * 1000;
-
- this.doodlebot.sendAudioData(uint8array);
- while (Date.now() < endTime) {
- await this.doodlebot.sendWebsocketCommand("d,O");
- await new Promise((res) => setTimeout(res, interval));
- await this.doodlebot.sendWebsocketCommand("d,N");
- await new Promise((res) => setTimeout(res, interval));
- }
-
} catch (error) {
- console.error("Error sending audio file:", error);
+ console.error("Error sending audio file:", error);
}
}
async isValidWavFile(file) {
const arrayBuffer = await file.arrayBuffer();
const dataView = new DataView(arrayBuffer);
-
+
// Check the "RIFF" chunk descriptor
const riff = String.fromCharCode(...new Uint8Array(arrayBuffer.slice(0, 4)));
if (riff !== "RIFF") {
console.error("Invalid WAV file: Missing RIFF header");
return false;
}
-
+
// Check the "WAVE" format
const wave = String.fromCharCode(...new Uint8Array(arrayBuffer.slice(8, 12)));
if (wave !== "WAVE") {
console.error("Invalid WAV file: Missing WAVE format");
return false;
}
-
+
// Check for "fmt " subchunk
const fmt = String.fromCharCode(...new Uint8Array(arrayBuffer.slice(12, 16)));
if (fmt !== "fmt ") {
console.error("Invalid WAV file: Missing fmt subchunk");
return false;
}
-
+
// Check for "data" subchunk
const dataIndex = arrayBuffer.byteLength - 8; // Approximate location
const dataChunk = String.fromCharCode(...new Uint8Array(arrayBuffer.slice(dataIndex, dataIndex + 4)));
@@ -1574,39 +1376,39 @@ blobToBase64(blob) {
console.error("Invalid WAV file: Missing data subchunk");
return false;
}
-
+
console.log("Valid WAV file");
return true;
}
-
+
async processAndSendAudio(buffer, endpoint, seconds) {
try {
- const wavBlob = await this.saveAudioBufferToWav(buffer);
- console.log(wavBlob);
- const wavFile = new File([wavBlob], "output.wav", { type: "audio/wav" });
-
- // // Create a temporary URL for the file
- // const url = URL.createObjectURL(wavFile);
-
- // // Create a temporary anchor element
- // const a = document.createElement("a");
- // a.href = url;
- // a.download = "output.wav";
- // document.body.appendChild(a);
-
- // // Trigger the download
- // a.click();
-
- // // Clean up
- // document.body.removeChild(a);
- // URL.revokeObjectURL(url);
- // const isValid = await this.isValidWavFile(wavFile);
- // if (!isValid) {
- // throw new Error("Generated file is not a valid WAV file");
- // }
- await this.sendAudioFileToChatEndpoint(wavFile, endpoint, wavBlob, seconds);
+ const wavBlob = await this.saveAudioBufferToWav(buffer);
+ console.log(wavBlob);
+ const wavFile = new File([wavBlob], "output.wav", { type: "audio/wav" });
+
+ // // Create a temporary URL for the file
+ // const url = URL.createObjectURL(wavFile);
+
+ // // Create a temporary anchor element
+ // const a = document.createElement("a");
+ // a.href = url;
+ // a.download = "output.wav";
+ // document.body.appendChild(a);
+
+ // // Trigger the download
+ // a.click();
+
+ // // Clean up
+ // document.body.removeChild(a);
+ // URL.revokeObjectURL(url);
+ // const isValid = await this.isValidWavFile(wavFile);
+ // if (!isValid) {
+ // throw new Error("Generated file is not a valid WAV file");
+ // }
+ await this.sendAudioFileToChatEndpoint(wavFile, endpoint, wavBlob, seconds);
} catch (error) {
- console.error("Error processing and sending audio:", error);
+ console.error("Error processing and sending audio:", error);
}
}
@@ -1619,18 +1421,18 @@ blobToBase64(blob) {
console.log("recording audio?")
const { context, buffer } = await this.doodlebot?.recordAudio(seconds);
console.log("finished recording audio");
-
+
// Display "thinking" while processing and waiting for response
await this.doodlebot?.display("clear");
await this.doodlebot?.displayText("thinking");
-
+
// Before sending audio to be played
await this.processAndSendAudio(buffer, endpoint, seconds);
-
+
// Display "speaking" when ready to speak
// await this.doodlebot?.display("clear");
// await this.doodlebot?.displayText("speaking");
-
+
// Wait a moment before clearing the display
await new Promise(resolve => setTimeout(resolve, 2000));
await this.doodlebot?.display("h");
@@ -1640,28 +1442,28 @@ blobToBase64(blob) {
try {
const modelUrl = this.modelArgumentToURL(url);
console.log('Loading model from URL:', modelUrl);
-
+
// Initialize prediction state if needed
this.predictionState[modelUrl] = {};
-
+
// Load and initialize the model
const { model, type } = await this.initModel(modelUrl);
this.predictionState[modelUrl].modelType = type;
this.predictionState[modelUrl].model = model;
-
+
// Update the current model reference
this.teachableImageModel = modelUrl;
-
- await this.indicate({
- type: "success",
- msg: "Model loaded successfully"
+
+ await this.indicate({
+ type: "success",
+ msg: "Model loaded successfully"
});
} catch (e) {
console.error('Error loading model:', e);
this.teachableImageModel = null;
- await this.indicate({
- type: "error",
- msg: "Failed to load model"
+ await this.indicate({
+ type: "error",
+ msg: "Failed to load model"
});
}
}
@@ -1670,7 +1472,7 @@ blobToBase64(blob) {
// Convert user-provided model URL/ID to the correct format
const endpointProvidedFromInterface = "https://teachablemachine.withgoogle.com/models/";
const redirectEndpoint = "https://storage.googleapis.com/tm-model/";
-
+
return modelArg.startsWith(endpointProvidedFromInterface)
? modelArg.replace(endpointProvidedFromInterface, redirectEndpoint)
: redirectEndpoint + modelArg + "/";
@@ -1684,12 +1486,12 @@ blobToBase64(blob) {
// First try loading as an image model
try {
const customMobileNet = await tmImage.load(modelURL, metadataURL);
-
+
// Check if it's actually an audio model
if ((customMobileNet as any)._metadata.hasOwnProperty('tfjsSpeechCommandsVersion')) {
const recognizer = await speechCommands.create("BROWSER_FFT", undefined, modelURL, metadataURL);
await recognizer.ensureModelLoaded();
-
+
// Setup audio listening
await recognizer.listen(async result => {
this.latestAudioResults = result;
@@ -1699,9 +1501,9 @@ blobToBase64(blob) {
invokeCallbackOnNoiseAndUnknown: true,
overlapFactor: 0.50
});
-
+
return { model: recognizer, type: this.ModelType.AUDIO };
- }
+ }
// Check if it's a pose model
else if ((customMobileNet as any)._metadata.packageName === "@teachablemachine/pose") {
const customPoseNet = await tmPose.load(modelURL, metadataURL);
@@ -1795,7 +1597,7 @@ blobToBase64(blob) {
try {
const imageStream = this.getImageStream();
if (!imageStream) {
- console.error("Failed to get image stream");
+ //console.error("Failed to get image stream");
return;
}
// const imageBitmap = await createImageBitmap(imageStream);
@@ -1880,21 +1682,21 @@ blobToBase64(blob) {
}
// Create indicator to show progress
- const indicator = await this.indicate({
+ const indicator = await this.indicate({
type: "warning",
- msg: `Capturing snapshots of ${imageClass}...`
+ msg: `Capturing snapshots of ${imageClass}...`
});
const snapshots: string[] = [];
const zip = new JSZip();
-
+
// Ensure we have video stream
this.imageStream ??= this.doodlebot?.getImageStream();
if (!this.imageStream) {
indicator.close();
- await this.indicate({
- type: "error",
- msg: "No video stream available"
+ await this.indicate({
+ type: "error",
+ msg: "No video stream available"
});
return;
}
@@ -1902,50 +1704,50 @@ blobToBase64(blob) {
// Capture a snapshot every 500ms
const interval = 100; // 500ms between snapshots
const iterations = (seconds * 1000) / interval;
-
+
for (let i = 0; i < iterations; i++) {
// Create a canvas to draw the current frame
const canvas = document.createElement('canvas');
canvas.width = this.imageStream.width;
canvas.height = this.imageStream.height;
const ctx = canvas.getContext('2d');
-
+
// Draw current frame to canvas
ctx.drawImage(this.imageStream, 0, 0);
-
+
// Convert to base64 and store
const dataUrl = canvas.toDataURL('image/jpeg');
snapshots.push(dataUrl);
-
+
// Add to zip file
const base64Data = dataUrl.replace(/^data:image\/jpeg;base64,/, "");
- zip.file(`${imageClass}_${i+1}.jpg`, base64Data, {base64: true});
-
+ zip.file(`${imageClass}_${i + 1}.jpg`, base64Data, { base64: true });
+
// Wait for next interval
await new Promise(resolve => setTimeout(resolve, interval));
}
// Generate zip file
- const content = await zip.generateAsync({type: "blob"});
-
+ const content = await zip.generateAsync({ type: "blob" });
+
// Create download link with image class in filename
const downloadUrl = URL.createObjectURL(content);
const link = document.createElement('a');
link.href = downloadUrl;
link.download = `${imageClass}_snapshots.zip`;
-
+
// Trigger download
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
-
+
// Cleanup
URL.revokeObjectURL(downloadUrl);
indicator.close();
-
- await this.indicate({
- type: "success",
- msg: `Captured ${snapshots.length} snapshots of ${imageClass}`
+
+ await this.indicate({
+ type: "success",
+ msg: `Captured ${snapshots.length} snapshots of ${imageClass}`
});
if (this.SOCIAL && Math.random() < this.socialness) {
@@ -1986,9 +1788,9 @@ blobToBase64(blob) {
resolve(null);
});
});
-
+
const durationMs = audio.duration * 1000; // duration is in seconds
-
+
// Convert blob to Uint8Array and send to Doodlebot
const array = await blob.arrayBuffer();
await this.doodlebot.sendAudioData(new Uint8Array(array));
diff --git a/extensions/src/doodlebot/package.json b/extensions/src/doodlebot/package.json
index 3f9121176..3ef8e11ac 100644
--- a/extensions/src/doodlebot/package.json
+++ b/extensions/src/doodlebot/package.json
@@ -24,6 +24,7 @@
"canvas": "^2.11.2",
"cubic-spline": "^3.0.3",
"curve-matcher": "^1.1.1",
+ "eventemitter3": "^5.0.1",
"events": "^3.3.0",
"jszip": "^3.10.1"
}
diff --git a/extensions/src/doodlebot/pnpm-lock.yaml b/extensions/src/doodlebot/pnpm-lock.yaml
index bd4c89c00..a443d7fd1 100644
--- a/extensions/src/doodlebot/pnpm-lock.yaml
+++ b/extensions/src/doodlebot/pnpm-lock.yaml
@@ -41,6 +41,9 @@ importers:
curve-matcher:
specifier: ^1.1.1
version: 1.1.1
+ eventemitter3:
+ specifier: ^5.0.1
+ version: 5.0.1
events:
specifier: ^3.3.0
version: 3.3.0
@@ -265,6 +268,9 @@ packages:
resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==}
engines: {node: '>=6'}
+ eventemitter3@5.0.1:
+ resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==}
+
events@3.3.0:
resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==}
engines: {node: '>=0.8.x'}
@@ -786,6 +792,8 @@ snapshots:
escalade@3.2.0: {}
+ eventemitter3@5.0.1: {}
+
events@3.3.0: {}
follow-redirects@1.15.9: {}
diff --git a/extensions/src/doodlebot/utils.ts b/extensions/src/doodlebot/utils.ts
index e27473cdf..58ca8f327 100644
--- a/extensions/src/doodlebot/utils.ts
+++ b/extensions/src/doodlebot/utils.ts
@@ -27,7 +27,7 @@ export const base64ToInt32Array = async (base64) => {
export const makeWebsocket = (ip: string, path: string) => new WebSocket(`wss://${ip}${path}`);
export const testWebSocket = (ip: string, port: string | number, timeoutSeconds?: number) => {
- const websocket = makeWebsocket(ip, port);
+ const websocket = makeWebsocket(ip, String(port));
return new Promise((resolve) => {
websocket.onopen = () => websocket.close();
websocket.onclose = (event) => resolve(event.wasClean);
@@ -36,3 +36,18 @@ export const testWebSocket = (ip: string, port: string | number, timeoutSeconds?
}
export const Max32Int = 2147483647;
+
+
+export const deferred = () => {
+ let resolve: (value: T | PromiseLike) => void;
+ let reject: (reason?: any) => void;
+
+ const promise = new Promise((res, rej) => {
+ resolve = res;
+ reject = rej;
+ });
+
+ return { promise, resolve: resolve!, reject: reject! };
+};
+
+export type Deferred = ReturnType>;
\ No newline at end of file
diff --git a/scratch-packages/scratch-gui b/scratch-packages/scratch-gui
index 5585d5eca..91c188acf 160000
--- a/scratch-packages/scratch-gui
+++ b/scratch-packages/scratch-gui
@@ -1 +1 @@
-Subproject commit 5585d5ecaeb0bcd50b4efabeb84811c924c17a1f
+Subproject commit 91c188acf6f2f1a9e267c8a5a61e196fb18bcc39
diff --git a/scratch-packages/scratch-vm b/scratch-packages/scratch-vm
index d79e45862..6b355943a 160000
--- a/scratch-packages/scratch-vm
+++ b/scratch-packages/scratch-vm
@@ -1 +1 @@
-Subproject commit d79e45862488335382fb03a2bb27032154c60f73
+Subproject commit 6b355943a631e17a2c9d37b4df3ea53a11506568