user_dtmf_receiving

Author: Haiping Chen

src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Utilities;
+using BotSharp.Core.Infrastructures;
 
 namespace BotSharp.Core.Realtime.Hooks;
 
@@ -29,6 +30,10 @@ public async Task OnFunctionExecuted(RoleDialogModel message)
         {
             return;
         }
+
+        // Clear cache to force to rebuild the agent instruction
+        Utilities.ClearCache();
+
         var routing = _services.GetRequiredService<IRoutingService>();
 
         message.Role = AgentRole.Function;

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -49,6 +49,9 @@ public async Task Listen(WebSocket userWebSocket,
             {
                 await _completer.AppenAudioBuffer(_conn.Data);
             }
+            else if (_conn.Event == "user_dtmf_receiving")
+            {
+            }
             else if (_conn.Event == "user_dtmf_received")
             {
                 await HandleUserDtmfReceived();

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -164,6 +164,11 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
             if (response.Type == "error")
             {
                 _logger.LogError($"{response.Type}: {receivedText}");
+                var error = JsonSerializer.Deserialize<ServerEventErrorResponse>(receivedText);
+                if (error?.Body.Type == "server_error")
+                {
+                    break;
+                }
             }
             else if (response.Type == "session.created")
             {

src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs

@@ -1,5 +1,4 @@
 using BotSharp.Abstraction.Realtime;
-using BotSharp.Abstraction.Realtime.Models;
 using BotSharp.Plugin.Twilio.Interfaces;
 using BotSharp.Plugin.Twilio.Models.Stream;
 using Microsoft.AspNetCore.Http;
@@ -100,6 +99,7 @@ await hub.Listen(webSocket, (receivedText) =>
                     }
                     else
                     {
+                        conn.Event = "user_dtmf_receiving";
                         conn.KeypadInputBuffer += dtmfResponse.Body.Digit;
                     }
                     break;
@@ -115,19 +115,29 @@ await hub.Listen(webSocket, (receivedText) =>
                     streamSid = response.StreamSid,
                     media = new { payload = message }
                 };
+
             conn.OnModelAudioResponseDone = () =>
                 new
                 {
                     @event = "mark",
                     streamSid = response.StreamSid,
                     mark = new { name = "responsePart" }
                 };
+
             conn.OnModelUserInterrupted = () =>
                 new
                 {
                     @event = "clear",
                     streamSid = response.StreamSid
                 };
+
+            /*if (response.Event == "dtmf")
+            {
+                // Send a Stop command to Twilio
+                string stopPlaybackCommand = "{ \"action\": \"stop_playback\" }";
+                var stopBytes = Encoding.UTF8.GetBytes(stopPlaybackCommand);
+                webSocket.SendAsync(new ArraySegment<byte>(stopBytes), WebSocketMessageType.Text, true, CancellationToken.None);
+            }*/
         });
     }
 }