0315

README.md

@@ -130,3 +130,5 @@ Read the docs: https://botsharp.readthedocs.io?wt.mc_id=AI-MVP-5005183
 If you feel that this project is helpful to you, please Star the project, we would be very grateful.
 
 Member project of [SciSharp STACK](https://github.com/SciSharp) which is the .NET based ecosystem of open-source software for mathematics, science, and engineering.
+
+Test by nick

src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs

@@ -23,7 +23,7 @@ Task Connect(RealtimeHubConnection conn,
     Task Disconnect();
 
     Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations);
-    Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true);
+    Task<string> UpdateSession(RealtimeHubConnection conn, bool turnDetection = true);
     Task InsertConversationItem(RoleDialogModel message);
     Task RemoveConversationItem(string itemId);
     Task TriggerModelInference(string? instructions = null);

src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs

@@ -1,5 +1,4 @@
 using BotSharp.Abstraction.Plugins;
-using BotSharp.Abstraction.Settings;
 using BotSharp.Core.Realtime.Hooks;
 using BotSharp.Core.Realtime.Services;
 using Microsoft.Extensions.Configuration;
@@ -15,12 +14,6 @@ public class RealtimePlugin : IBotSharpPlugin
 
     public void RegisterDI(IServiceCollection services, IConfiguration config)
     {
-        services.AddScoped(provider =>
-        {
-            var settingService = provider.GetRequiredService<ISettingService>();
-            return settingService.Bind<RealtimeModelSettings>("RealtimeModel");
-        });
-
         services.AddScoped<IRealtimeHub, RealtimeHub>();
         services.AddScoped<IConversationHook, RealtimeConversationHook>();
     }

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -98,18 +98,15 @@ private async Task ConnectToModel(WebSocket userWebSocket)
         await _completer.Connect(_conn, 
             onModelReady: async () => 
             {
-                // Not TriggerModelInference, waiting for user utter.
-                await _completer.UpdateSession(_conn);
-
-                // Push dialogs into model context
-                foreach (var message in dialogs)
+                if (states.ContainsState("init_audio_file"))
                 {
-                    await _completer.InsertConversationItem(message);
+                    await _completer.UpdateSession(_conn, turnDetection: true);
                 }
-
-                // Trigger model inference if there is no audio file in the conversation
-                if (!states.ContainsState("init_audio_file"))
+                else
                 {
+                    // Control initial session, prevent initial response interruption
+                    await _completer.UpdateSession(_conn, turnDetection: false);
+
                     if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
                     {
                         await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
@@ -118,6 +115,10 @@ await _completer.Connect(_conn,
                     {
                         await _completer.TriggerModelInference("Reply based on the conversation context.");
                     }
+
+                    // Start turn detection
+                    await Task.Delay(1000 * 8);
+                    await _completer.UpdateSession(_conn, turnDetection: true);
                 }
             },
             onModelAudioDeltaReceived: async (audioDeltaData, itemId) =>

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs

@@ -52,9 +52,6 @@ public class RealtimeSessionBody
 
 public class RealtimeSessionTurnDetection
 {
-    [JsonPropertyName("interrupt_response")]
-    public bool InterruptResponse { get; set; } = true;
-
     /// <summary>
     /// Milliseconds
     /// </summary>

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -290,7 +290,7 @@ public async Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogMod
         return session;
     }
 
-    public async Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true)
+    public async Task<string> UpdateSession(RealtimeHubConnection conn, bool turnDetection = true)
     {
         var convService = _services.GetRequiredService<IConversationService>();
         var conv = await convService.GetConversation(conn.ConversationId);
@@ -317,8 +317,6 @@ public async Task<string> UpdateSession(RealtimeHubConnection conn, bool interru
         var words = new List<string>();
         HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
 
-        var realitmeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
-
         var sessionUpdate = new
         {
             type = "session.update",
@@ -337,18 +335,22 @@ public async Task<string> UpdateSession(RealtimeHubConnection conn, bool interru
                 ToolChoice = "auto",
                 Tools = functions,
                 Modalities = [ "text", "audio" ],
-                Temperature = Math.Max(options.Temperature ?? realitmeModelSettings.Temperature, 0.6f),
-                MaxResponseOutputTokens = realitmeModelSettings.MaxResponseOutputTokens,
+                Temperature = Math.Max(options.Temperature ?? 0f, 0.6f),
+                MaxResponseOutputTokens = 512,
                 TurnDetection = new RealtimeSessionTurnDetection
                 {
-                    InterruptResponse = interruptResponse,
-                    Threshold = realitmeModelSettings.TurnDetection.Threshold,
-                    PrefixPadding = realitmeModelSettings.TurnDetection.PrefixPadding,
-                    SilenceDuration = realitmeModelSettings.TurnDetection.SilenceDuration
+                    Threshold = 0.9f,
+                    PrefixPadding = 300,
+                    SilenceDuration = 800
                 }
             }
         };
 
+        if (!turnDetection)
+        {
+            sessionUpdate.session.TurnDetection = null;
+        }
+
         await HookEmitter.Emit<IContentGeneratingHook>(_services, async hook =>
         {
             await hook.OnSessionUpdated(agent, instruction, functions);

src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj

@@ -19,6 +19,12 @@
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-outbound_phone_call.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-twilio-hangup_phone_call.fn.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-twilio-outbound_phone_call.fn.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
   </ItemGroup>
 
   <ItemGroup>
@@ -33,8 +39,4 @@
     <ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 
-  <ItemGroup>
-    <Folder Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\" />
-  </ItemGroup>
-
 </Project>

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs

@@ -429,15 +429,6 @@ public async Task<FileContentResult> GetSpeechFile([FromRoute] string conversati
         return result;
     }
 
-    [ValidateRequest]
-    [HttpPost("twilio/voice/hang-up")]
-    public async Task<TwiMLResult> Hangup(ConversationalVoiceRequest request)
-    {
-        var twilio = _services.GetRequiredService<TwilioService>();
-        var response = twilio.HangUp("twilio/bye.mp3");
-        return TwiML(response);
-    }
-
     [ValidateRequest]
     [HttpPost("twilio/voice/status")]
     public async Task<ActionResult> PhoneCallStatus(ConversationalVoiceRequest request)

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs

@@ -1,34 +1,28 @@
-using BotSharp.Abstraction.Routing;
 using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
 using Twilio.Rest.Api.V2010.Account;
+using Task = System.Threading.Tasks.Task;
 
 namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions;
 
 public class HangupPhoneCallFn : IFunctionCallback
 {
     private readonly IServiceProvider _services;
     private readonly ILogger<HangupPhoneCallFn> _logger;
-    private readonly TwilioSetting _twilioSetting;
 
     public string Name => "util-twilio-hangup_phone_call";
     public string Indication => "Hangup";
 
     public HangupPhoneCallFn(
         IServiceProvider services,
-        ILogger<HangupPhoneCallFn> logger,
-        TwilioSetting twilioSetting)
+        ILogger<HangupPhoneCallFn> logger)
     {
         _services = services;
         _logger = logger;
-        _twilioSetting = twilioSetting;
     }
 
     public async Task<bool> Execute(RoleDialogModel message)
     {
         var args = JsonSerializer.Deserialize<HangupPhoneCallArgs>(message.FunctionArgs);
-
-        var routing = _services.GetRequiredService<IRoutingService>();
-        var conversationId = routing.Context.ConversationId;
         var states = _services.GetRequiredService<IConversationStateService>();
         var callSid = states.GetState("twilio_call_sid");
 
@@ -39,20 +33,20 @@ public async Task<bool> Execute(RoleDialogModel message)
             return false;
         }
 
-        if (args.AnythingElseToHelp)
-        {
-            message.Content = "Tell me how I can help.";
-        }
-        else
+        message.Content = args.GoodbyeMessage;
+
+        _ = Task.Run(async () =>
         {
+            await Task.Delay(args.GoodbyeMessage.Split(' ').Length * 400);
+            // Have to find the SID by the phone number
             var call = CallResource.Update(
-                url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?conversation-id={conversationId}"),
+                status: CallResource.UpdateStatusEnum.Completed,
                 pathSid: callSid
             );
 
-            message.Content = "The call is ending.";
+            message.Content = "The call has been ended.";
             message.StopCompletion = true;
-        }
+        });
 
         return true;
     }

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs

@@ -66,15 +66,15 @@ public async Task<bool> Execute(RoleDialogModel message)
         var recordingStatusUrl = $"{_twilioSetting.CallbackHost}/twilio/recording/status?conversation-id={newConversationId}";
 
         // Generate initial assistant audio
-        string initAudioFile = null;
+        string initAudioUrl = null;
         if (!string.IsNullOrEmpty(args.InitialMessage))
         {
             var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
             var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage);
-            initAudioFile = "intial.mp3";
-            fileStorage.SaveSpeechFile(newConversationId, initAudioFile, data);
+            initAudioUrl = "intial.mp3";
+            fileStorage.SaveSpeechFile(newConversationId, initAudioUrl, data);
 
-            statusUrl += $"&init-audio-file={initAudioFile}";
+            statusUrl += $"&init-audio-file={initAudioUrl}";
         }
 
         // Set up process URL streaming or synchronous
@@ -88,17 +88,13 @@ public async Task<bool> Execute(RoleDialogModel message)
             await sessionManager.SetAssistantReplyAsync(newConversationId, 0, new AssistantMessage
             {
                 Content = args.InitialMessage,
-                SpeechFileName = initAudioFile
+                SpeechFileName = initAudioUrl
             });
 
             processUrl += "/voice/init-outbound-call";
         }
 
-        processUrl += $"?conversation-id={newConversationId}";
-        if (!string.IsNullOrEmpty(initAudioFile))
-        {
-            processUrl += $"&init-audio-file={initAudioFile}";
-        }
+        processUrl += $"?conversation-id={newConversationId}&init-audio-file={initAudioUrl}";
 
         // Make outbound call
         var call = await CallResource.CreateAsync(

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs

@@ -4,6 +4,6 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
 
 public class HangupPhoneCallArgs
 {
-    [JsonPropertyName("anything_else_to_help")]
-    public bool AnythingElseToHelp { get; set; } = true;
+    [JsonPropertyName("goodbye_message")]
+    public string? GoodbyeMessage { get; set; }
 }

src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json

@@ -1,19 +1,15 @@
 {
   "name": "util-twilio-hangup_phone_call",
-  "description": "Call this function if the user wants to end the phone call or conversation",
+  "description": "Call this function if the user wants to end the phone call",
   "visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}",
   "parameters": {
     "type": "object",
     "properties": {
-      "reason": {
+      "goodbye_message": {
         "type": "string",
-        "description": "The reason why user wants to end the phone call."
-      },
-      "anything_else_to_help": {
-        "type": "boolean",
-        "description": "Check if user has anything else to help."
+        "description": "A polite closing statement for ending a conversation."
       }
     },
-    "required": [ "reason", "anything_else_to_help" ]
+    "required": [ "goodbye_message" ]
   }
 }

src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid

@@ -1,3 +1,3 @@
 {% if channel == 'phone' %}
-** If user wants to end the phone call or conversation, ask user if there is anything else to help. If not, end the phone call.
+** Please call util-twilio-hangup_phone_call if user wants to end the phone call.
 {% endif %}