diff --git a/_locales/en/messages.json b/_locales/en/messages.json index b964be761a..ce5262b2ab 100644 --- a/_locales/en/messages.json +++ b/_locales/en/messages.json @@ -27,6 +27,14 @@ "message": "End hands-free conversation", "description": "The tooltip to display on the call button when a call is in progress." }, + "momentaryCallPaused": { + "message": "Hold to speak, or click to return to hands-free mode", + "description": "The tooltip to display on the call button when a call is in progress and momentary mode is paused." + }, + "momentaryCallListening": { + "message": "Release when finished speaking", + "description": "The tooltip to display on the call button when a call is in progress and momentary mode is active." + }, "callInterruptible": { "message": "Tap to interrupt", "description": "The tooltip to display on the call button when a call is in progress and manual interruption is possible." @@ -198,6 +206,16 @@ "example": "Pi" } } + }, + "microphoneIsMuted": { + "message": "The microphone is muted...", + "description": "Message displayed when the 'momentary' mode is enabled but paused.", + "placeholders": { + "chatbot": { + "content": "$1", + "example": "Pi" + } + } }, "toggleThemeToDarkMode": { "message": "Switch to dark mode", diff --git a/src/ButtonModule.js b/src/ButtonModule.js index 619b8d53fd..c12c681027 100644 --- a/src/ButtonModule.js +++ b/src/ButtonModule.js @@ -11,6 +11,8 @@ import callIconSVG from "./icons/call.svg"; import callStartingIconSVG from "./icons/call-starting.svg"; import hangupIconSVG from "./icons/hangup.svg"; import interruptIconSVG from "./icons/interrupt.svg"; +import momentaryPausedIconSVG from "./icons/momentary_paused.svg"; +import momentaryListeningIconSVG from "./icons/momentary_listening.svg"; import hangupMincedIconSVG from "./icons/hangup-minced.svg"; import lockIconSVG from "./icons/lock.svg"; import unlockIconSVG from "./icons/unlock.svg"; @@ -23,6 +25,7 @@ import { IconModule } from "./icons/IconModule.ts"; import { ImmersionStateChecker } from "./ImmersionServiceLite.ts"; import { GlowColorUpdater } from "./buttons/GlowColorUpdater.js"; import { openSettings } from "./popup/popupopener.ts"; +import { ButtonHelper } from "./buttons/ButtonHelper.js"; class ButtonModule { /** @@ -31,20 +34,17 @@ class ButtonModule { */ constructor(chatbot) { this.icons = new IconModule(); + this.buttonHelper = new ButtonHelper(); this.userPreferences = UserPreferenceModule.getInstance(); this.chatbot = chatbot; this.immersionService = new ImmersionService(chatbot); this.glowColorUpdater = new GlowColorUpdater(); - this.sayPiActor = StateMachineService.actor; // the Say, Pi state machine this.screenLockActor = StateMachineService.screenLockActor; // Binding methods to the current instance this.registerOtherEvents(); // track the frequency of bug #26 this.submissionsWithoutAnError = 0; - - // track whether a call is active, so that new button instances can be initialized correctly - this.callIsActive = false; } registerOtherEvents() { @@ -189,7 +189,7 @@ class ButtonModule { button.id = id; button.type = "button"; button.className = `saypi-control-button rounded-full bg-cream-550 enabled:hover:bg-cream-650 tooltip mini ${className}`; - button.setAttribute("aria-label", label); + this.setAriaLabelOf(button, label); const svgElement = createSVGElement(icon); button.appendChild(svgElement); @@ -204,7 +204,7 @@ class ButtonModule { createExitButton(container, position = 0) { const button = this.createIconButton({ id: 'saypi-exit-button', - label: getMessage("exitImmersiveModeLong"), + label: "exitImmersiveModeLong", icon: exitIconSVG, onClick: () => ImmersionService.exitImmersiveMode(), className: 'saypi-exit-button' @@ -217,7 +217,7 @@ class ButtonModule { createEnterButton(container, position = 0) { const button = this.createIconButton({ id: 'saypi-enter-button', - label: getMessage("enterImmersiveModeLong"), + label: "enterImmersiveModeLong", icon: maximizeIconSVG, onClick: () => this.immersionService.enterImmersiveMode(), className: 'saypi-enter-button' @@ -234,9 +234,10 @@ class ButtonModule { */ createControlButton(options) { const { shortLabel, longLabel = shortLabel, icon, onClick, className = '' } = options; + const button = createElement("a", { className: `${className} maxi saypi-control-button tooltip flex h-16 w-16 flex-col items-center justify-center rounded-xl text-neutral-900 hover:bg-neutral-50-hover hover:text-neutral-900-hover active:bg-neutral-50-tap active:text-neutral-900-tap gap-0.5`, - ariaLabel: longLabel, + ariaLabel: getMessage(longLabel), onclick: onClick, }); @@ -245,7 +246,7 @@ class ButtonModule { const labelDiv = createElement("div", { className: "t-label", - textContent: shortLabel, + textContent: getMessage(shortLabel), }, ); button.appendChild(labelDiv); @@ -254,8 +255,8 @@ class ButtonModule { createImmersiveModeButton(container, position = 0) { const button = this.createControlButton({ - shortLabel: getMessage("enterImmersiveModeShort"), - longLabel: getMessage("enterImmersiveModeLong"), + shortLabel: "enterImmersiveModeShort", + longLabel: "enterImmersiveModeLong", icon: immersiveIconSVG, onClick: () => this.immersionService.enterImmersiveMode(), className: 'immersive-mode-button' @@ -266,9 +267,8 @@ class ButtonModule { } createSettingsButton(container, position = 0) { - const label = getMessage("extensionSettings"); const button = this.createControlButton({ - shortLabel: label, + shortLabel: "extensionSettings", icon: settingsIconSVG, onClick: () => openSettings(), className: 'settings-button' @@ -281,7 +281,7 @@ class ButtonModule { createMiniSettingsButton(container, position = 0) { const button = this.createIconButton({ id: 'saypi-settingsButton', - label: getMessage("extensionSettings"), + label: "extensionSettings", icon: settingsIconSVG, onClick: () => openSettings(), className: 'settings-button' @@ -297,30 +297,20 @@ class ButtonModule { button.type = "button"; button.classList.add("call-button", "saypi-button", "tooltip"); button.classList.add(...this.chatbot.getExtraCallButtonClasses()); - if (this.callIsActive) { + if (this.buttonHelper.isCallActive()) { this.callActive(button); } else { this.callInactive(button); } addChild(container, button, position); - if (this.callIsActive) { + if (this.buttonHelper.isCallActive()) { // if the call is active, start the glow animation once added to the DOM AnimationModule.startAnimation("glow"); } return button; } - updateCallButtonColor(color) { - const callButton = document.getElementById("saypi-callButton"); - // find first path element descendant of the call button's svg element child - const path = callButton?.querySelector("svg path"); - if (path) { - // set the fill color of the path element - path.style.fill = color; - } - } - /** * * @param { isSpeech: number; notSpeech: number } probabilities @@ -329,71 +319,80 @@ class ButtonModule { this.glowColorUpdater.updateGlowColor(probabilities.isSpeech); } - updateCallButton(callButton, svgIcon, label, onClick, isActive = false) { - if (!callButton) { - callButton = document.getElementById("saypi-callButton"); - } - if (callButton) { - // Remove all existing child nodes - while (callButton.firstChild) { - callButton.removeChild(callButton.firstChild); - } - - const svgElement = createSVGElement(svgIcon); - callButton.appendChild(svgElement); - - callButton.setAttribute("aria-label", label); - callButton.onclick = onClick; - callButton.classList.toggle("active", isActive); - } - this.callIsActive = isActive; - } - callStarting(callButton) { - const label = getMessage("callStarting"); - this.updateCallButton(callButton, callStartingIconSVG, label, () => - this.sayPiActor.send("saypi:hangup") - ); + this.buttonHelper.updateCallButton({ + button: callButton, + icon: callStartingIconSVG, + label: "callStarting", + clickEventName: "saypi:hangup", + }); } callActive(callButton) { - const label = getMessage("callInProgress"); - this.updateCallButton( - callButton, - hangupIconSVG, - label, - () => this.sayPiActor.send("saypi:hangup"), - true - ); + this.buttonHelper.updateCallButton({ + button: callButton, + icon: hangupIconSVG, + label: "callInProgress", + clickEventName: "saypi:hangup", + longPressEventName: "saypi:momentaryListen", + isCallActive: true, + }); } + callMomentary(callButton) { + this.buttonHelper.updateCallButton({ + button: callButton, + icon: momentaryListeningIconSVG, + label: "momentaryCallListening", + clickEventName: "saypi:momentaryStop", + longReleaseEventName: "saypi:momentaryPause", + + isCallActive: true, + }); + } + + pauseMomentary(callButton) { + this.buttonHelper.updateCallButton({ + button: callButton, + icon: momentaryPausedIconSVG, + label: "momentaryCallPaused", + clickEventName: "saypi:momentaryStop", + longPressEventName: "saypi:momentaryListen", + isCallActive: true, + }); + } + callInterruptible(callButton) { const handsFreeInterruptEnabled = this.userPreferences.getCachedAllowInterruptions(); + if (!handsFreeInterruptEnabled) { - const label = getMessage("callInterruptible"); - this.updateCallButton( - callButton, - interruptIconSVG, - label, - () => { - this.sayPiActor.send("saypi:interrupt"); - }, - true - ); + this.buttonHelper.updateCallButton({ + button: callButton, + icon: interruptIconSVG, + label: "callInterruptible", + clickEventName: "saypi:interrupt", + isCallActive: true, + }); } } callInactive(callButton) { - const label = getMessage("callNotStarted", this.chatbot.getName()); - this.updateCallButton(callButton, callIconSVG, label, () => - this.sayPiActor.send("saypi:call") - ); + this.buttonHelper.updateCallButton({ + button: callButton, + icon: callIconSVG, + label: "callNotStarted", + labelArgument: this.chatbot.getName(), + clickEventName: "saypi:call", + }); } callError(callButton) { - const label = getMessage("callError"); - this.updateCallButton(callButton, hangupMincedIconSVG, label, null); + this.buttonHelper.updateCallButton({ + button: callButton, + icon: hangupMincedIconSVG, + label: "callError", + }); } disableCallButton() { @@ -435,14 +434,18 @@ class ButtonModule { return button; } + setAriaLabelOf(button, labelName) { + const label = getMessage(labelName); + button.setAttribute("aria-label", label); + } + createUnlockButton(container) { - const label = getMessage("unlockButton"); const button = document.createElement("button"); button.id = "saypi-unlockButton"; button.type = "button"; button.className = "lock-button saypi-control-button rounded-full bg-cream-550 enabled:hover:bg-cream-650 tooltip"; - button.setAttribute("aria-label", label); + this.setAriaLabelOf(button, "unlockButton"); button.appendChild(createSVGElement(unlockIconSVG)); if (container) { container.appendChild(button); diff --git a/src/FullscreenModule.ts b/src/FullscreenModule.ts index 416edc4640..bce4e16c8f 100644 --- a/src/FullscreenModule.ts +++ b/src/FullscreenModule.ts @@ -8,12 +8,24 @@ import { isMobileDevice } from "./UserAgentModule"; const focusActor = interpret(focusMachine); const tickInterval = 1000; var ticker: NodeJS.Timeout; -const userInputEvents = ["mousemove", "click", "keypress"]; +const userInputEvents = ["mousemove", "keypress"]; function handleUserInput() { focusActor.send({ type: "blur" }); } +function handleMouseDown() { + if(document.fullscreenEnabled) { + focusActor.send({ type: "pause" }); + } +} + +function handleMouseUp() { + if(document.fullscreenEnabled) { + focusActor.send({ type: "resume" }); + } +} + function startFocusModeListener() { focusActor.start(); ticker = setInterval(() => { @@ -23,6 +35,8 @@ function startFocusModeListener() { for (const event of userInputEvents) { document.addEventListener(event, handleUserInput); } + document.addEventListener("mousedown", handleMouseDown); + document.addEventListener("mouseup", handleMouseUp); } function stopFocusModeListener() { diff --git a/src/audio/AudioModule.js b/src/audio/AudioModule.js index 0c3bc66bf1..114248eaec 100644 --- a/src/audio/AudioModule.js +++ b/src/audio/AudioModule.js @@ -375,6 +375,10 @@ export default class AudioModule { // soft stop recording inputActor.send("stopRequested"); }); + EventBus.on("audio:quickStopRecording", function (e) { + // soft stop recording but quickly + inputActor.send("quickStopRequested"); + }); // audio input (recording) events (pass media recorder events -> audio input machine actor) EventBus.on("audio:dataavailable", (detail) => { inputActor.send({ type: "dataAvailable", ...detail }); diff --git a/src/buttons/ButtonHelper.js b/src/buttons/ButtonHelper.js new file mode 100644 index 0000000000..70b00f436f --- /dev/null +++ b/src/buttons/ButtonHelper.js @@ -0,0 +1,201 @@ +import getMessage from "../i18n.ts"; +import StateMachineService from "../StateMachineService.js"; +import { createSVGElement } from "../dom/DOMModule.ts"; + +class ButtonUpdater{ + + constructor() { + this.longPressMilliseconds = 500; + this.clickStartTime = 0; + this.isLongClickEngaged = false; + this.isMouseDown = false; + + this.sayPiActor = StateMachineService.actor; + + // track whether a call is active, so that new button instances can be initialized correctly + this.callIsActive = false; + + this.clickEventName = ""; + this.longDownEventName = ""; + this.longUpEventName = ""; + + this.isMouseOutHandled = false; + this.areButtonClicksConfigured = false; + + this.iconSvgCache = new Array(); + this.areListenersEnabled = true; + } + + removeChildrenFrom(button) { + while (button.firstChild) { + button.removeChild(button.firstChild); + } + } + + setAriaLabelOf(button, labelName, labelArg) { + const label = labelArg? getMessage(labelName, labelArg) : getMessage(labelName); + button.setAttribute("aria-label", label); + } + + toggleActiveState(callButton, isActive) { + callButton.classList.toggle("active", isActive); + } + + isShortClick() { + const clickDuration = (Date.now() - this.clickStartTime); + return this.clickStartTime == 0 || clickDuration < this.longPressMilliseconds; + } + + sendEvent(eventName) { + if (eventName) { + this.sayPiActor.send(eventName); + } + } + + sendClickEvent() { + this.sendEvent(this.clickEventName); + } + + sendLongDownEvent() { + this.sendEvent(this.longDownEventName); + } + + sendLongUpEvent() { + this.sendEvent(this.longUpEventName); + } + + onDown() { + if (!this.areListenersEnabled) { + return; + } + this.isMouseDown = true; + this.clickStartTime = Date.now(); + window.setTimeout( () => { + if (this.isMouseDown === true) { + this.sendLongDownEvent(); + this.isLongClickEngaged = true; + } + }, this.longPressMilliseconds); + } + + onUp() { + if (!this.areListenersEnabled) { + return; + } + this.isMouseDown = false; + if (this.isShortClick()) { + this.sendClickEvent(); + } else if (this.isLongClickEngaged) { + this.sendLongUpEvent(); + this.isLongClickEngaged = false; + } + } + + onMouseMovedOut() { + if (this.isMouseOutHandled && this.isLongClickEngaged && this.areListenersEnabled) { + this.sendLongUpEvent(); + this.isLongClickEngaged = false; + } + } + + isUsingTouchEvents() { + return typeof(window.ontouchstart) != 'undefined'; + } + + setupButtonListeners(button) { + if (this.isUsingTouchEvents()) { + button.addEventListener('touchstart',() => { + this.onDown(); + }); + button.addEventListener('touchend', ev => { + ev.preventDefault(); + this.onUp(); + }); + button.addEventListener('touchcancel', () => { + this.onUp(); + }); + } else { + button.onmousedown = () => this.onDown() ; + button.onmouseup = () => this.onUp(); + button.addEventListener("mouseout", () => this.onMouseMovedOut()); + } + } + + updateButtonRoles(options) { + let { button, clickEventName = "", longDownEventName = "", longUpEventName = "", isMouseOutHandled = false } = options; + + if (!this.areButtonClicksConfigured) { + this.setupButtonListeners(button); + this.areButtonClicksConfigured = true; + } + this.clickEventName = clickEventName + this.longDownEventName = longDownEventName + this.longUpEventName = longUpEventName; + this.isMouseOutHandled = isMouseOutHandled; + } + + isCallActive() { + return this.callIsActive; + } + + getSvgElement(icon, key) { + if (!this.iconSvgCache.includes(key)) { + this.iconSvgCache[key] = createSVGElement(icon); + } + return this.iconSvgCache[key]; + } + + changeIcon(button, icon, iconKey, isSvgOverwritten) { + const svgElement = this.getSvgElement(icon, iconKey); + if (!isSvgOverwritten || !this.isUsingTouchEvents()) { + this.removeChildrenFrom(button); + button.appendChild(svgElement); + } else if (button.firstChild) { + button.firstChild.appendChild(svgElement); + } else { + button.appendChild(svgElement); + } + } + + updateCallButton(options) { + let {button, icon, label, labelArgument, clickEventName, longPressEventName, longReleaseEventName, isCallActive = false} = options; + if (!button) { + button = document.getElementById("saypi-callButton"); + } + if (button) { + this.areListenersEnabled = false; + const iconKey = label; + const isSvgOverwritten = longReleaseEventName? true : false; + const isMouseOutProcessed = isSvgOverwritten; + + this.changeIcon(button, icon, iconKey, isSvgOverwritten); + this.setAriaLabelOf(button, label, labelArgument); + this.toggleActiveState(button, isCallActive); + this.callIsActive = isCallActive; + this.updateButtonRoles({ + button: button, + clickEventName: clickEventName, + longDownEventName: longPressEventName, + longUpEventName: longReleaseEventName, + isMouseOutHandled: isMouseOutProcessed, + }); + this.areListenersEnabled = true; + } + } +} + +class ButtonHelper { + constructor() { + this.buttonUpdater = new ButtonUpdater(); + } + + updateCallButton(options) { + this.buttonUpdater.updateCallButton(options); + } + + isCallActive() { + return this.buttonUpdater.isCallActive(); + } +} + +export { ButtonHelper } \ No newline at end of file diff --git a/src/events/EventModule.js b/src/events/EventModule.js index 77de5a52cc..c05ac926ba 100644 --- a/src/events/EventModule.js +++ b/src/events/EventModule.js @@ -19,6 +19,10 @@ const AUDIO_DEVICE_RECONNECT = "saypi:audio:reconnect"; const END_CALL = "saypi:hangup"; const SESSION_ASSIGNED = "saypi:session:assigned"; const UI_SHOW_NOTIFICATION = "saypi:ui:show-notification"; +const MOMENTARY_LISTEN = "saypi:momentaryListen"; +const MOMENTARY_PAUSE = "saypi:momentaryPause"; +const MOMENTARY_STOP = "saypi:momentaryStop"; +const MOMENTARY_SUBMIT_TRANSCRIPTIONS = "saypi:momentarySubmitTranscriptions"; /** * The EventModule translates events sent on the EventBus to StateMachine events, @@ -76,6 +80,10 @@ export default class EventModule { PI_STOPPED_SPEAKING, PI_FINISHED_SPEAKING, END_CALL, + MOMENTARY_LISTEN, + MOMENTARY_PAUSE, + MOMENTARY_STOP, + MOMENTARY_SUBMIT_TRANSCRIPTIONS, ].forEach((eventName) => { EventBus.on(eventName, () => { actor.send(eventName); diff --git a/src/icons/momentary_listening.svg b/src/icons/momentary_listening.svg new file mode 100644 index 0000000000..cc859447a1 --- /dev/null +++ b/src/icons/momentary_listening.svg @@ -0,0 +1,108 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/icons/momentary_paused.svg b/src/icons/momentary_paused.svg new file mode 100644 index 0000000000..0c8d3118de --- /dev/null +++ b/src/icons/momentary_paused.svg @@ -0,0 +1,108 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/state-machines/AudioInputMachine.ts b/src/state-machines/AudioInputMachine.ts index a186eb2b9b..d15f460152 100644 --- a/src/state-machines/AudioInputMachine.ts +++ b/src/state-machines/AudioInputMachine.ts @@ -313,7 +313,7 @@ function tearDownRecording(): void { microphone.pause(); listening = false; stream.getTracks().forEach((track) => track.stop()); - microphone.destroy(); //added by JAC + microphone.destroy(); } else { console.log("microphone does not exist!"); } @@ -324,6 +324,7 @@ interface AudioInputContext { waitingToStop: boolean; waitingToStart: boolean; recordingStartTime: number; + isQuickStopRequested: boolean; } type AudioInputEvent = @@ -331,6 +332,7 @@ type AudioInputEvent = | { type: "release" } | { type: "start" } | { type: "stopRequested" } + | { type: "quickStopRequested" } | { type: "dataAvailable"; blob: Blob; duration: number } | { type: "stop" } | { type: "error.platform"; data: any }; @@ -347,6 +349,7 @@ export const audioInputMachine = createMachine< waitingToStop: false, waitingToStart: false, recordingStartTime: 0, + isQuickStopRequested: false, }, states: { released: { @@ -412,6 +415,13 @@ export const audioInputMachine = createMachine< stopRequested: { target: "pendingStop", description: "Stop gracefully.", + actions: [ assign({isQuickStopRequested: false}) ], + }, + + quickStopRequested: { + target: "pendingStop", + description: "Stop gracefully and quickly.", + actions: [ assign({isQuickStopRequested: true}) ], }, stop: { @@ -436,7 +446,7 @@ export const audioInputMachine = createMachine< type: "prepareStop", }, after: { - "5000": [ + stopRecordingDelay: [ { target: "#audioInput.acquired.stopped", actions: ["stopIfWaiting"], @@ -459,7 +469,7 @@ export const audioInputMachine = createMachine< }, }, }, - + stopped: { entry: assign({ waitingToStop: false }), always: { @@ -609,7 +619,11 @@ export const audioInputMachine = createMachine< return context.waitingToStart === true; }, }, - delays: {}, + delays: { + stopRecordingDelay: (context) => { + return context.isQuickStopRequested === true ? 1000 : 5000; + } + }, } ); interface OverconstrainedError extends DOMException { diff --git a/src/state-machines/FocusMachine.ts b/src/state-machines/FocusMachine.ts index e01528bf0e..3bb5d254e6 100644 --- a/src/state-machines/FocusMachine.ts +++ b/src/state-machines/FocusMachine.ts @@ -17,6 +17,8 @@ type FocusContext = { }; type TickEvent = { type: "tick"; time_ms: number }; type BlurEvent = { type: "blur" }; +type PauseEvent = { type: "pause" }; +type ResumeEvent = { type: "resume" }; export const machine = createMachine( { @@ -45,6 +47,19 @@ export const machine = createMachine( tick: { actions: "incrementInactivityTime", }, + pause: { + target: "Paused" + } + }, + }, + Paused: { + description: + "The machine is not waiting for user activity", + on: { + resume: { + actions: "resetInactivityTime", + target: "#focusMachine", + }, }, }, Focused: { diff --git a/src/state-machines/SayPiMachine.ts b/src/state-machines/SayPiMachine.ts index 56fe266ac1..230856bc15 100644 --- a/src/state-machines/SayPiMachine.ts +++ b/src/state-machines/SayPiMachine.ts @@ -74,6 +74,10 @@ type SayPiEvent = | { type: "saypi:submit" } | { type: "saypi:promptReady" } | { type: "saypi:call" } + | { type: "saypi:momentaryListen" } + | { type: "saypi:momentaryPause" } + | { type: "saypi:momentaryStop" } + | { type: "saypi:momentarySubmitTranscriptions" } | { type: "saypi:callReady" } | { type: "saypi:callFailed" } | { type: "saypi:hangup" } @@ -92,6 +96,9 @@ interface SayPiContext { timeUserStoppedSpeaking: number; defaultPlaceholderText: string; sessionId?: string; + isMomentaryEnabled: boolean; + isMomentaryActive: boolean; + hasUserSpoken: boolean; } // Define the state schema @@ -122,6 +129,7 @@ type SayPiStateSchema = { }; }; }; + momentaryPaused: {}; responding: { states: { piThinking: {}; @@ -200,14 +208,17 @@ function getChatbotDefaultPlaceholder(): string { } const machine = createMachine( { - /** @xstate-layout N4IgpgJg5mDOIC5SwIYE8AKBLAdFgdigMYAuWAbmAMSpoAOWCdATgPYC2dJASmChGgDaABgC6iUHVawsZVvgkgAHogDMAdgAcAThzrt64aoAsAJgBsAVk3HLlgDQg0iAIwvTwnJs0ntmt34uBgC+wY602HiEpBTUtAwIRCgANski4kggUjJyCpkqCNqmjs6Fxrqm1uqm6naVmsLqoeHokQTEZJQ06AkMAMp0fADWBFDpitmyWPKKBcZulnra5tqWRqqW7g5OriaqXqrm6i6ax8KW6hrNIBG4Sal9JCjMZPhQ3fSM98m8-EJiE2kUxm+UQtmMOBcxhs1m8xn0+m2pWhi2MxnMphq2hc5hcwmMh2utxw30ez1e73ijAAFig3gBXOjjTKTXKzRCaYo7BB41Y4NanDaXczmYQuSxE1p3FLJMkvUYfBLfABiKCwyUgzMkQLZoIQaIhUJh3hsCNWJVc2n2llU2jtm3x-ltksw0oeT3lbyoSlgTxIYBwKAAZv7mAAKDzCKMASg+kVJHopWqyOumeVAc0shuhthN8IM5u5Jk8dtUWxqVisErCNylOGSWF9YHwCqpTCwABVqQQRm9k6y0+yEKpDvsEVHrCtzMZzhbhxtzJDvEc3KcjsYXZEG02W162-1Bihe2MASzUyCM2oNmODBPNFOZ0i1O51Dh5mtqqtTD4fJvcNv-V3SkekYchGywAAjDV+3PdNlFcEVNH5Sd-CsTEbFUOdxQsPQdGWdRDEqLY-3rRtANbECEBQekIGmRJ5HwMBSE1U9tRyQc9RxdEcFtdEHytfQsIJTwR3XXiNHRFwSIA5sKM+KiaLo5gmIYpiSBg9iL3gnlp1fNDbHxKMcRMOdr1fQ5UL8Az0WksjZL3SjYDgGR5Co2AZCgRiIA04E4IKLiIXKTE0RXe8DFMtES3FFxDHUGcXDLasWldUid1GHBlKIVhmFohz5NpBkmVYlNNL8q8bSWQxzjC6dZ25SN9lUc46ltW1Kg3GtiRkoCMpUnL0vwVgSAGYY5ISeknOYZUCEbalIBGo9Rh83VL3nCrx2qh86tKTF8RwEVyg2TQs05VRbLSt5eqy-rLsG4bD2PRVGAmsBmAW49lo41btEafbTmhb97xRdQ51MaL9unbwPHMVRMW0c7yMuzLstyqAcBet6HrG57JseVg6EGCB3qW4qBy0gpzG8HA7TtOELD8ZZQcqUwIfKHRp2ncUktrFLuvS5GbrRjHiby8bcZIfHCZFsYXAyNjfKHSmkJpvwbHp9mmeEFmuKtawTsxbmursnqsvwShPTR4giHpdh6WSFAKSoCB5ADAhyFYIYA3YV6YAAeS4LB2DIrAiE+8nXGqt91Csco3DMZnQdQg4lcMMsak0BH7LR03zYpQMiGt237cd162GYHA6GLoNsvYHBveYP2A6D30Q7DsqeXOXQoVsC4THFKNMPq6pGp8FZhAaeFTHhTOTfkXP0qtm27Yd7GEBIZg6VgIhmEgli5ZKhW9QMfYwdOHw4rtI4Qfqi5FitfFoSa0V4TOzq6z5y6c9evPF6LlfRcYOvTe29d6qnVHvQEpUhxgxwraGwVY1h+EHjtWq1MoR2kaGYZYr9kpbmNulL+Ft86F2Xo7NsQD8Bbx3hBSAABRTgJB-j7zJu3GBi44HojsIgnwidhCLnmOULQCUwYiiaG-Xm+DP5z2-gvAuS9i4Kh9H6AMwZQxhlgPSCCzcXL4AACJgHtmgWMRsLrZ2kUQ3+pCSbMNgkOKEMUcCn0uJYWmeIwamQHpCKw6wEpaCaobd+kizFmxkZdDRWjZCOzbkOUSN4qqTmWI+UyMVXzYnKFrYQP1rAzwIeYvOFCqGQVXgUkBNDvKk1sXqG0EInG1BHOffEiccQ8Q0I0Iw95-D4hyVIkJRCSnUOKRvShpSwBgI1OUmxUCqkmEcScAiNpVANOMKDbQaJHE-QSm1XE5wxG4P-EEkkeT0r9KKQAteQzCllPoVwJhkDD6rWqbM04dTFmtOWUWLWi5TA-S1hfTk34AkSNMTgUu2UyQkAmiC5gZdYDel9A7FRIZXphjWDGOM+zgWgren6SFWLYDRL1N80w+wjgFntBcBoyDECVH0G+dBpx8LEsJOIyIylYBSHwKjJ6CACpQEZAS1a8xxRLBWGsEcmxKhYXFEhGcqdMRQgxDFEibKOVcrbMLLGfYKlTMFSOV8QVcyWG-LrJ8w4CT7EyZPUUp1yjKrgKq9KDAuw9lXgeUaWrJn3O0vCLW1MbBFHHqfQ4HiVh6GhEEVZ6JVZ2vZfIVGFdOzdnwI9fcWAADqO8kzaq9XMQifqgqBpOMGosU8WZT05EENwYMCQxodZdN1i0zn9AlgTeamqTyepWtpDEpkoaOIrAYMGhwzAdT2b1WNnLHVYGlty5tks23uplp2r63qFgitWOsCVpq3C-R+qnXM05sTmFrXGqdM7U3TRbLAOaRN20Cu0ka0yVoWbVHmXic4xKswnsnfW6d7buUasXfegoNo+THVcRcY4dhTIWGVugkcYobSbFHTzVl9rT2-ozVMJtf6gPZq7SB1pLSRSBr4YemDJw6XQn0JiDQ2SWW4BVRhtGDAsNkMonO1tEA2PWLuQRxA1Re1lj0BWAk8d8SrO-fGjGABJfAoZmCMnY-JYWLapZ3vwyuwjhhiPWuhuRostpFwdPlR4f1GcGPjrrULSacmFNKdXqp+dt6gOyz41pjkXJSiHGjl4GBpp4T6A2FJs9-62wEHs1wYDiAXHwhwFrTkdh-DWC1qZbZkImrGu8JcEKIXf3nsohF0uSnBBubPDq7tYMXD8hhhiTJ3zbSmo2GsDLCXVY5ZstcQaEA4CKFuO58OCAAC05gsJ8PizOcemwYawhMCRdoMRKADfbnR6rZhK0YlLCkrCNRGo+ZnLif1uzUNullImUYy2hxmDnOt+LlaPxWBxDobpUBLtH1G9yaVr4p5pwIlrWEL2roowu+VnNrg0RYR+i+4SsJJyxcBwLeNd1pZvdWm4BKPF8SUxhtUDw8JQYnBZmk0sdTLj3gR31aTuN22o+0kELz1KFVhrLNHaO5RyjHssx-YJ883i0-8uUUG0IZX+IIscHQdhAeEJ-nIv+FJ+eIGxIsU+lKcu3ypQgb5wrvlRh8DSnwUkucHOl+lcJQcSDy9B-xnk+hjPuGo2iAiOIPs7S-BDNYUJbT6BOJzsd3PDm9PyRc0pIP5bW6CNfV3lRHF4QuFxfQ8MjeYuhWCnF8Arcec1+GniVkNB+OqKa9Cuh3xohOCOm0gOsXgshYNZg7AUgK813UHPGECLn0lUWGKqJ+7j3q0ayvKfsUO1xYP9PYfM-M10BwvP7fC+3YEf84lUa0QD7LtX2AUKYU4BOTQsZkBG80vMrntvGgO8oIx++dYZYhUoZMYjNGVe0+b+yhvoORBaGD4P5BlvBIT8F9Bm3mgmsB4KnCznlq9hnoNmYDoG+EUIasas1onNYI4gDM1HAbUOAQms6smqHgfNbuavqnAXrAgdBp3sJrTERJkkUFfJgQ2seI3ilizEFDoG0rtIWN5t+K+AlPiGTpksdDgidlZsxgmjxnzpAe3IItVgPE1NUKcCAckj9LAf8jDGYFoJ1mOkxj+jZq9HZsVgHGIePlAd4LoJTPiDruKLpKZIstViYC-COO4BgmIqEEAA */ + /** @xstate-layout N4IgpgJg5mDOIC5SwIYE8AKBLAdFgdigMYAuWAbmAMSpoAOWCdATgPYC2dJASmChGgDaABgC6iUHVawsZVvgkgAHogDMAdgAcAThzrt64aoAsAJgBsAVk3HLlgDQg0iAIwvTwnJs0ntmt34uBgC+wY602HiEpBTUtAwIRCgANski4kggUjJyCpkqCNqmjs6Fxrqm1uqm6naVmsLqoeHokQTEZJQ06AkMAMp0fADWBFDpitmyWPKKBcZulnra5tqWRqqW7g5OriaqXqrm6i6ax8KW6hrNIBG4Sal9JCjMZPhQ3fSM98m8-EJiE2kUxm+UQtmMOBcxhs1m8xn0+m2pWhi2MxnMphq2hc5hcwmMh2utxw30ez1e73ijAAFig3gBXOjjTKTXKzRCaYo7BB41Y4NanDaXczmYQuSxE1p3FLJMkvUYfBLfABiKCwyUgzMkQLZoIQaIhUJh3hsCNWJVc2n2llU2jtm3x-ltksw0oeT3lbyoSlgTxIYBwKAAZv7mAAKDzCKMASg+kVJHopWqyOumeVAc0shuhthN8IM5u5Jk8dtUWxqVisErCNylOGSWF9YHwCqpTCwABVqQQRm9k6y0+yEKpDvsEVHrCtzMZzhbhxtzJDvEc3KcjsYXZEG02W162-1Bihe2MASzUyCM2oNmODBPNFOZ0i1O51Dh5mtqqtTD4fJvcNv-V3SkekYchGywAAjDV+3PdNlFcEVNH5Sd-CsTEbFUOdxQsPQdGWdRDEqLY-3rRtANbECEBQekIGmRJ5HwMBSE1U9tRyQc9RxdEcFtdEHytfQsIJTwR3XXiNHRFwSIA5sKM+KiaLo5gmIYpiSBg9iL3gnlp1fNDbHxKMcRMOdr1fQ5UL8Az0WksjZL3SjYDgGR5Co2AZCgRiIA04E4IKLiIXKTE0RXe8DFMtES3FFxDHUGcXDLasWldUid1GHBlKIVhmFohz5NpBkmVYlNNL8q8bSWQxzjC6dZ25SN9lUc46ltW1Kg3GtiRkoCMpUnK5ISdgOGbD00AAGTs-AfN1S8eUsO031MZZ7w0TF8XUOdTEit8mrRHxhFFeFVFstK3l6rL+rO-BWBIAZhgGxh6Sc5hlQIRtqUgO6j1GaaONmkcKvHaqHzq0o1ohEVyg2TQs05Y7Orrbr0sy7LcqgHBrtuw9j0VR7nq+49fq0gptEaHBzFOaFv3vFENvq6LyenbwPHMVRMW0E7yLOlHLvRp6wGYAmHoQfnBZIVg6EGCAhb7YqB2JxAKaQu07ThCw-GWTbKlMRnyh0adp3FJLaxSpHub6tGcFFmXgPk63xclz7sZ+lwMjY3yhyVnAVb8Gx1f1rXhB1rirWsWHMWNrrJvSob2BG540F4C60dxhBY-j5hMGopyibKhBatMoJ9hFNYoVtfQTnMTn7PR9P8FGpPUeFuvRowbOwEEV3AVKz2Z1MtmkKWoyCQLSvq56luE8b3nU8nzO2-5wRTDdkqPb1Aui2hczLCIkeK-vceY+G+up4t9LMZt1Pred2WV-lvP0MXLZNrxTwtrLGpCNhQ+zrnrP+YgLPY+o1HgS1zkOLMxhC5WkZqXPe2ID4I1NtHX+wCE4LycoAtsf8Jo7nAXqSB-d5pLGHuXBBVckFbhQejLK+BKCenRsQIg9J2D0mSCgCkVAIDyADAQcgrAhgBjjswGAAB5LgWB2BkSwEQfBs08TWDfOoKw5Q3BmG1ptVCBwlaGA-loH+ND5D0IpIGIgzDWHsM4QLNgzAcB0EsUGbK7AcDCLERIqRvoZFyO0go3QUJbAXBMOKKMmF6rVEaj4FYwgGjwi2k0Sh-5qEkiMQLExTCWFsI4cLEgzA6SwCIMwSCLE76wSHAYfYpgThaAknaI4dMwYXEWFafE0ImqHUuAY5JdDUnpXSRYrJeUEg5LyQUopqp1TFO7mvWalScK2hsFWNYfhQlg1qt7KEdpGhmGWPDZKVDTqGO6Qw0x5jMmcLbMM-A+TCkQUgAAUU4CQf4JSe56lmYueZ6I7BLJ8Jog6b4NlaASpUkU8S9mJIOV04xvSzEZMsQqH0foAzBlDGGWA9IIIeJcvgAAImAdhaBYxR0hbQ6FZ0+lnJ+nLUpnF5ivkqYKeagQg4uFMiEyEVh1gJWqecTppKelnXRZi2QnDvEFFEjeKqk5liPkLgRdZ5Qg7CFJtYPlKTjmXOuZBbJuSrmjNud5alrzZo2ghAygiNpVA+EMFA+qWiRyxSMPefw+I1VHJMZq-VOqRk3LAOMjUhqXnTO0qanA5ragjmtfiTa2g0RhtJglNquJzhgpNvsrmhyyXo09Tc71erfUQAeVwZ5UyZohpMGGqpEarUaGjUWIOi4h5Bzin4dCkdEZJOsdlMkJAno4C7cwWA3pfQcORSGAWYY1gxjjBCjN-bmA2J7X2gdsAxVqCMjgMUZYNjKtxFoTadgkLRJxFmBKNo7SpuJMpWAUh8ApzbAVKAjI136gWEsFYawAbP25AsJCM5dGYihBiGKJFr23vvZRa+91b6lr+tpAkGg3xFFzDvHQGwnzDgQ5u2N1RRRw3KKBuA4H0oMC7D2YWB5oMniDWWuYhFvY2CKNEhlhw2UrD0NCIIsb0S+0Ize+QltSPdnwDjfcWAADqhSkxGuDXRoODGgrMZOKxosW0dZbU5EENwlSCR8eI2dSj31BmMH6A7KWNsX0YlMszMNFYDCVMOGYDq4Ler8bvSRrAl8xOgMdtLG+Ywu5nmNfBt9+gP3rE2JULCeJXyk10bmac2IKEubAwJjzXnKIMFei2WAH0-NUZfTvUyVodbVAta-HeGxnNptwKl9zBnPP+avvjfzhXbSLBhqrcUBEjamQsMrDZI4xQ2k2NVq9RG0sNYy-JAgoZmCMnUjJ2jiB5rwk3d+So1hxQNC5KUQ4eJIRNW-L7S4IU9OTfRoZ0TlFZvWIW53GjcGCjAZcPyVmGJlVLXa6ZOwngEpBx0N4U7NkEmuf05diTUmKONYK0tp7K3a08UOHhlmiW+snABVvIopgNCqtB3VwTkOpjGfbD5qWknifUdgwrBA1RrNlj0BWAk6j8SxvO-V9GAB3NUFJlTZWwB2VgPnpu9E82Zp2sPHtaT24YJ+R3AeChCo4AoNoj1AsqOzy23PKd8+YALoXDtL6ItHYGcd4Yp1ErrAT9K2vef887AbiWFm4c09V5u9XGHDiKP+8doHJgQcpYmxzq2z0ACS9c7sSJJ-bCW5nWsu7zjaQwSORTMYOmjostpFzOsAx4RjmhNfpVFuHubC3hYx98xZwL7tlsIE5KZI42fZmmnhPoDYoQazXQgHARQtxqd5wALT+Kwv8wyUIjCJYwu2lK7QYiUH70OXHr2zBaYxKWGK9TXA1EaocOKB0TjlEvXWBM5JRgL71GYOcK-N1aY-FYHEOgDHn9mgPzWP7HNhoJERMUAOpKg7NujDzGjM-j4miFhKTKVl-ueisLGkYJ0kAefDdDbCAf5G4I1PiBTKzNUB4PCC-JyAqqWBGpcIgi5gAedE3GdFBkZlACgZaLtogDpouHFGWMosouUOUMljVqlHOn-NPMAUFrJq4A0MIFhEcLoMwZULhniH-qQUkrwWfFdEgf5rQTyDYJoOAexu-C+F-N4J0vIcnEXi1lRioQfuofTNCJ-nojoQXv-nIWgvPO3BACoQPtCP3EEAClOg0EUDvG6lmiYeUJtNCH+ktM6vFloGNh2iSuqmkrCv0hSCodiIsAyg0BoCYI0isgwYkfGlGD4JIT4DIVwWQfysckKlIiQPEQIbXjFMsF4O4FvGiD1iKDGjvLAmKPAmPLYVEe6ulDmtqm8CYeFPVN4WGnhBcFxPoBzJ0XOgOkuvAJUfDggJtkhJcBvktFtO4DUFrBTJupyMqmYA6HFNPumjXPOoun6H2tdMwOwCkCoRYEtHoCwZ+OsZiJvsOOKBCMeuuBYAdCQYUZ2gut2ucbAKcdlHMTXgsd8UwY8WsfMC8YEVGDsUEDjiPDYBEcgpCjMUCSCYOjgL0bcv6pALcSog8aschhsa8ZUEUIiTAVTN+HAVMScZiRwsugCTiVIkQHcqyUSVmCSQRDCeSZtLUK+FtpgacEaCOIXv0fMTTi4WYWDDDIPK0uKKzGgdOJKRDmRiJmftKXnAhq+EFChsduhoXAzqrN-naBYAROqXYjDtQSodYPJkFDoI0EHB4IWHtt+K+P9kdF4TDLslwdbg1hThUeCTTofq9iEk1NUKcB4K8WWKTEhpyBYCYDUAstabbqMLrvrsLsoTqUOCIUWLLodgDidv7pweNm5pbMXhHgumXlKaGbqd4LoBTPiEPMqbgZnv4DxA0SOB-JsvEqEEAA */ context: { transcriptions: {}, isTranscribing: false, + isMomentaryEnabled: false, + isMomentaryActive: false, lastState: "inactive", userIsSpeaking: false, timeUserStoppedSpeaking: 0, defaultPlaceholderText: "", + hasUserSpoken: false, }, id: "sayPi", initial: "inactive", @@ -257,6 +268,7 @@ const machine = createMachine( { type: "callStartingPrompt", }, + assign({ isMomentaryEnabled: false }), ], exit: [ { @@ -352,7 +364,7 @@ const machine = createMachine( }, }, { - type: "listenPrompt", + type: "listenOrPausedPrompt", }, ], exit: [ @@ -389,7 +401,10 @@ const machine = createMachine( animation: "userSpeaking", }, }, - assign({ userIsSpeaking: true }), + assign({ + userIsSpeaking: true, + hasUserSpoken: true, + }), { type: "cancelCountdownAnimation", }, @@ -443,9 +458,60 @@ const machine = createMachine( description: 'Disable the VAD microphone.\n Aka "call" Pi.\n Stops active listening.', }, + "saypi:momentaryListen": { + actions: [ + assign({ + isMomentaryEnabled: true, + isMomentaryActive: true, + hasUserSpoken: false, + }), + { + type: "momentaryHasStarted", + }, + { + type: "listenPrompt", + }, + ], + description: + 'Enable Momentary Mode. Now recording will only stop if the user releases the button.', + }, + "saypi:momentaryPause": [ + { + actions: [ + assign({ + isMomentaryActive: false, + }), + { + type: "momentaryPausedPrompt", + }, + { + type: "momentaryHasPaused", + }, + ] + }, + { + target: "#sayPi.listening.converting.submitting", + cond: "readyToSubmitFromMomentary", + }, + ], + "saypi:momentaryStop": { + actions: [ + assign({ + isMomentaryActive: false, + isMomentaryEnabled: false, + }), + { + type: "momentaryHasStopped", + }, + { + type: "listenPrompt" + }, + ], + description: + 'Enable Momentary Mode. Now recording will only stop if the user releases the button.', + }, }, }, - converting: { initial: "accumulating", states: { @@ -459,7 +525,7 @@ const machine = createMachine( description: "Submit combined transcript to Pi.", }, }, - entry: { + entry: { type: "draftPrompt", }, invoke: { @@ -544,6 +610,22 @@ const machine = createMachine( description: "Out of sequence empty response from the /transcribe API", }, + "saypi:momentaryPause": { + actions: [ + assign({ + isMomentaryActive: false, + }), + { + type: "momentaryHasPaused", + }, + { + type: "stopAnimation", + params: { + animation: "glow", + }, + }, + ], + }, }, }, submitting: { @@ -588,6 +670,22 @@ const machine = createMachine( }, description: "Successfully transcribed user audio to text.", }, + "saypi:momentaryPause": { + actions: [ + assign({ + isMomentaryActive: false, + }), + { + type: "momentaryHasPaused", + }, + { + type: "stopAnimation", + params: { + animation: "glow", + }, + }, + ], + }, "saypi:transcribeFailed": { target: [ "accumulating", @@ -714,6 +812,47 @@ const machine = createMachine( type: "parallel", }, + momentaryPaused: { + // this state obviates the modification of the user_interrupting state for when momentary mode is enabled + description: "In momentary mode and the button has been released, so the microphone is ignoring input", + entry:[ + { + type: "pauseAudio", + }, + { + type: "momentaryReturnsToPaused", + }, + { + type: "momentaryPausedPrompt", + }, + ], + on: { + "saypi:momentaryListen": { + actions: [ + assign({ isMomentaryActive: true }), + { + type: "momentaryHasStarted" + }, + ], + target: "#sayPi.listening.recording", + description: 'Returning to the standard recording mode.', + }, + "saypi:momentaryStop": { + actions: [ + assign({ isMomentaryEnabled: false }), + { + type: "momentaryHasStopped" + }, + { + type: "listenPrompt" + }, + ], + target: "#sayPi.listening.recording", + description: 'Returning to the standard recording mode.', + }, + }, + }, + responding: { initial: "piThinking", on: { @@ -785,11 +924,19 @@ const machine = createMachine( piSpeaking: { on: { "saypi:piStoppedSpeaking": [ + { + target: "#sayPi.momentaryPaused", + cond: + { + type: "isMomentaryEnabled", + }, + }, { target: "#sayPi.listening", - cond: { - type: "wasListening", - }, + cond: + { + type: "wasListening", + }, }, { target: "#sayPi.inactive", @@ -798,9 +945,16 @@ const machine = createMachine( }, }, ], - "saypi:piFinishedSpeaking": { - target: "#sayPi.listening", - }, + "saypi:piFinishedSpeaking": [ + { + cond: "isMomentaryEnabled", + target: "#sayPi.momentaryPaused", + }, + { + target: "#sayPi.listening", + cond: "isMomentaryDisabled" + }, + ], "saypi:userSpeaking": { target: "userInterrupting", cond: { @@ -811,6 +965,19 @@ const machine = createMachine( }, description: "The user starting speaking while Pi was speaking.", + }, + "saypi:momentaryListen": { + target: "#sayPi.listening.recording", + actions:[ + { + type: "pauseAudio", + }, + { + type: "momentaryHasStarted", + } + ], + description: + "The user pushed the momentary button while while Pi was speaking.", }, "saypi:interrupt": [ { @@ -818,6 +985,12 @@ const machine = createMachine( description: `The user has forced an interruption, i.e. tapped to interrupt Pi, during a call.`, actions: "pauseAudio", cond: "wasListening", + }, + { + target: "#sayPi.momentaryPaused", + description: `The user has forced an interruption while momentary mode was enabled, i.e. tapped to interrupt Pi, during a call.`, + actions: "pauseAudio", + cond: "wasListeningWithMomentary", }, { target: "#sayPi.inactive", @@ -882,15 +1055,30 @@ const machine = createMachine( }, waitingForPiToStopSpeaking: { on: { - "saypi:piStoppedSpeaking": { - target: "userInterrupting", - }, - }, + "saypi:piStoppedSpeaking":[ + { + cond: "isMomentaryEnabled", + actions: "momentaryHasPaused", + target: "#sayPi.momentaryPaused", + }, + { + target: "userInterrupting", + cond: "isMomentaryDisabled" + }, + ]}, after: { - 500: { - target: "userInterrupting", - description: "Fallback transition after 500ms if piStoppedSpeaking event does not fire.", - }, + 500: [ + { + cond: "isMomentaryEnabled", + actions: "momentaryHasPaused", + target: "#sayPi.momentaryPaused", + }, + { + target: "userInterrupting", + cond: "isMomentaryDisabled", + description: "Fallback transition after 500ms if piStoppedSpeaking event does not fire.", + }, + ] }, description: "Interrupt requested. Waiting for Pi to stop speaking before recording.", }, @@ -927,6 +1115,20 @@ const machine = createMachine( description: "User has spoken.", }, ], + "saypi:momentaryListen" : [ + { + actions: [ + assign({ + isMomentaryEnabled: true, + isMomentaryActive: true, + }), + { + type: "momentaryHasStarted" + }, + ], + target: "#sayPi.listening.recording", + }, + ], }, entry: [ { @@ -1007,7 +1209,7 @@ const machine = createMachine( }); } }, - + acquireMicrophone: (context, event) => { // warmup the microphone on idle in mobile view, // since there's no mouseover event to trigger it @@ -1028,7 +1230,7 @@ const machine = createMachine( pauseRecording: (context, event) => { EventBus.emit("audio:input:stop"); }, - + pauseRecordingIfInterruptionsNotAllowed: (context, event) => { const handsFreeInterrupt = userPreferences.getCachedAllowInterruptions(); @@ -1083,11 +1285,24 @@ const machine = createMachine( }, listenPrompt: () => { - const message = getMessage("assistantIsListening", chatbot.getName()); - if (message) { - getPromptOrNull()?.setMessage(message); + setPromptMessage("assistantIsListening", chatbot); + }, + + listenOrPausedPrompt: (context: SayPiContext) => { + const isMomentaryPaused = context.isMomentaryEnabled && !context.isMomentaryActive; + if (isMomentaryPaused) { + setPromptMessage("microphoneIsMuted"); + } else { + setPromptMessage("assistantIsListening", chatbot); } }, + + momentaryPausedPrompt: (context: SayPiContext) => { + context.hasUserSpoken + ? setPromptMessage("assistantIsThinking", chatbot) + : setPromptMessage("microphoneIsMuted"); + }, + callStartingPrompt: () => { const message = getMessage("callStarting"); if (message) { @@ -1096,43 +1311,36 @@ const machine = createMachine( getPromptOrNull()?.setMessage(message); } }, + thinkingPrompt: () => { - const message = getMessage("assistantIsThinking", chatbot.getName()); - if (message) { - getPromptOrNull()?.setMessage(message); - } + setPromptMessage("assistantIsThinking", chatbot); }, + writingPrompt: () => { - const message = getMessage("assistantIsWriting", chatbot.getName()); - if (message) { - getPromptOrNull()?.setMessage(message); - } + setPromptMessage("assistantIsWriting", chatbot); }, + speakingPrompt: (context: SayPiContext) => { const handsFreeInterrupt = userPreferences.getCachedAllowInterruptions(); - const message = handsFreeInterrupt - ? getMessage("assistantIsSpeaking", chatbot.getName()) - : getMessage( - "assistantIsSpeakingWithManualInterrupt", - chatbot.getName() - ); - if (message) { - getPromptOrNull()?.setMessage(message); - } + + handsFreeInterrupt + ? setPromptMessage("assistantIsSpeaking", chatbot) + : setPromptMessage("assistantIsSpeakingWithManualInterrupt", chatbot); }, + interruptingPiPrompt: () => { - const message = getMessage( - "userStartedInterrupting", - chatbot.getName() - ); - if (message) { - getPromptOrNull()?.setMessage(message); - } + setPromptMessage("userStartedInterrupting", chatbot); + }, + + pushToTalkPrompt: () => { + setPromptMessage("pressAndHoldToSpeak"); }, + clearPrompt: (context: SayPiContext) => { getPromptOrNull()?.setMessage(context.defaultPlaceholderText); }, + draftPrompt: (context: SayPiContext) => { const text = mergeService .mergeTranscriptsLocal(context.transcriptions) @@ -1150,62 +1358,78 @@ const machine = createMachine( callIsStarting: () => { buttonModule.callStarting(); }, + callFailedToStart: () => { buttonModule.callInactive(); audibleNotifications.callFailed(); - }, + }, + callNotStarted: () => { if (buttonModule) { // buttonModule may not be available on initial load buttonModule.callInactive(); } }, + callHasStarted: () => { buttonModule.callActive(); audibleNotifications.callStarted(); EventBus.emit("session:started"); }, + callInterruptible: () => { buttonModule.callInterruptible(); }, + callInterruptibleIfListening: (context: SayPiContext) => { if (context.lastState === "listening") { buttonModule.callInterruptible(); } }, + callContinues: () => { buttonModule.callActive(); }, + callHasEnded: () => { visualNotifications.listeningStopped(); buttonModule.callInactive(); audibleNotifications.callEnded(); EventBus.emit("session:ended"); }, + callHasErrors: () => { buttonModule.callError(); }, + callHasNoErrors: () => { buttonModule.callActive(); }, + disableCallButton: () => { buttonModule.disableCallButton(); }, + enableCallButton: () => { buttonModule.enableCallButton(); }, + cancelCountdownAnimation: () => { visualNotifications.listeningStopped(); }, + activateAudioOutput: () => { audioControls.activateAudioOutput(true); }, + requestWakeLock: () => { requestWakeLock(); }, + releaseWakeLock: () => { releaseWakeLock(); }, + notifySentMessage: (context: SayPiContext, event: SayPiEvent) => { const delay_ms = Date.now() - context.timeUserStoppedSpeaking; const submission_delay_ms = lastSubmissionDelay; @@ -1214,18 +1438,46 @@ const machine = createMachine( wait_time_ms: submission_delay_ms, }); }, - clearPendingTranscriptionsAction: () => { + + clearPendingTranscriptionsAction: (context: SayPiContext) => { // discard in-flight transcriptions. Called after a successful submission clearPendingTranscriptions(); }, + clearTranscriptsAction: assign({ transcriptions: () => ({}), }), + pauseAudio: () => { EventBus.emit("audio:output:pause"); }, + resumeAudio: () => { EventBus.emit("audio:output:resume"); + }, + + momentaryHasStarted: () => { + buttonModule.callMomentary(); + AnimationModule.startAnimation("glow"); + EventBus.emit("audio:input:reconnect"); + }, + + momentaryHasPaused: (context: SayPiContext,) => { + buttonModule.pauseMomentary(); + AnimationModule.stopAnimation("glow"); + AnimationModule.stopAnimation("userSpeaking"); + EventBus.emit(context.hasUserSpoken ? "audio:stopRecording" : "audio:quickStopRecording"); + }, + + momentaryReturnsToPaused: () => { + AnimationModule.stopAnimation("glow"); + buttonModule.pauseMomentary(); + }, + + momentaryHasStopped: () => { + buttonModule.callActive(); + AnimationModule.startAnimation("glow"); + EventBus.emit("audio:input:reconnect"); }, }, services: {}, @@ -1237,6 +1489,7 @@ const machine = createMachine( } return false; }, + hasNoAudio: (context: SayPiContext, event: SayPiEvent) => { if (event.type === "saypi:userStoppedSpeaking") { event = event as SayPiSpeechStoppedEvent; @@ -1248,6 +1501,7 @@ const machine = createMachine( } return false; }, + submissionConditionsMet: ( context: SayPiContext, event: SayPiEvent, @@ -1255,21 +1509,36 @@ const machine = createMachine( ) => { const { state } = meta; const autoSubmitEnabled = userPreferences.getCachedAutoSubmit(); - return autoSubmitEnabled && readyToSubmit(state, context); + return autoSubmitEnabled && readyToSubmit(state, context) && !context.isMomentaryActive; }, + wasListening: (context: SayPiContext) => { - return context.lastState === "listening"; + return context.lastState === "listening" && !context.isMomentaryEnabled; }, + wasInactive: (context: SayPiContext) => { return context.lastState === "inactive"; }, + interruptionsAllowed: (context: SayPiContext) => { const allowInterrupt = userPreferences.getCachedAllowInterruptions(); return allowInterrupt; - }, - interruptionsNotAllowed: (context: SayPiContext) => { - const allowInterrupt = userPreferences.getCachedAllowInterruptions(); - return !allowInterrupt; + }, + + isMomentaryEnabled: (context: SayPiContext) => { + return context.isMomentaryEnabled; + }, + + isMomentaryDisabled: (context: SayPiContext) => { + return !context.isMomentaryEnabled; + }, + + readyToSubmitFromMomentary: (context: SayPiContext) => { + return readyToSubmitOnAllowedState(true, context); + }, + + wasListeningWithMomentary: (context: SayPiContext) => { + return context.lastState === "listening" && context.isMomentaryEnabled; }, }, delays: { @@ -1319,12 +1588,16 @@ const machine = createMachine( // Capture the delay for analytics events lastSubmissionDelay = finalDelay; - + return finalDelay; }, + momentarySubmissionDelay: (context: SayPiContext, event: SayPiEvent) => { + return 100; + }, }, } ); + function readyToSubmitOnAllowedState( allowedState: boolean, context: SayPiContext @@ -1334,10 +1607,21 @@ function readyToSubmitOnAllowedState( const ready = allowedState && !empty && !pending; return ready; } + function provisionallyReadyToSubmit(context: SayPiContext): boolean { const allowedState = !(context.userIsSpeaking || context.isTranscribing); // we don't have access to the state, so we read from a copy in the context (!DRY) return readyToSubmitOnAllowedState(allowedState, context); } + +function setPromptMessage(messageName: string, chatbot?: Chatbot) : void { + const message = chatbot + ? getMessage(messageName, chatbot.getName()) + : getMessage(messageName); + if (message) { + getPromptOrNull()?.setMessage(message); + } +} + function readyToSubmit( state: State, context: SayPiContext