From 4c0435486144ae0bdb7434158d912281117168c2 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 10 Feb 2026 15:44:08 -0800 Subject: [PATCH 01/92] Realtime Client Proposal --- .../Realtime/DelegatingRealtimeSession.cs | 79 + .../Realtime/IRealtimeClient.cs | 32 + .../Realtime/IRealtimeSession.cs | 59 + .../Realtime/NoiseReductionOptions.cs | 28 + .../Realtime/RealtimeAudioFormat.cs | 36 + ...timeClientConversationItemCreateMessage.cs | 35 + ...timeClientInputAudioBufferAppendMessage.cs | 32 + ...timeClientInputAudioBufferCommitMessage.cs | 23 + .../Realtime/RealtimeClientMessage.cs | 29 + .../RealtimeClientResponseCreateMessage.cs | 86 + .../Realtime/RealtimeContentItem.cs | 54 + .../Realtime/RealtimeServerErrorMessage.cs | 40 + ...imeServerInputAudioTranscriptionMessage.cs | 53 + .../Realtime/RealtimeServerMessage.cs | 34 + .../Realtime/RealtimeServerMessageType.cs | 148 ++ .../RealtimeServerOutputTextAudioMessage.cs | 56 + .../RealtimeServerResponseCreatedMessage.cs | 85 + ...RealtimeServerResponseOutputItemMessage.cs | 42 + .../Realtime/RealtimeSessionKind.cs | 23 + .../Realtime/RealtimeSessionOptions.cs | 138 ++ .../SemanticVoiceActivityDetection.cs | 21 + .../Realtime/ServerVoiceActivityDetection.cs | 36 + .../Realtime/TranscriptionOptions.cs | 38 + .../Realtime/VoiceActivityDetection.cs | 23 + .../Tools/ToolChoiceMode.cs | 28 + .../UsageDetails.cs | 46 + ....Extensions.AI.Evaluation.Reporting.csproj | 2 +- .../Microsoft.Extensions.AI.OpenAI.csproj | 5 + .../OpenAIClientExtensions.cs | 17 + .../OpenAIRealtimeClient.cs | 80 + .../OpenAIRealtimeSession.cs | 1922 +++++++++++++++++ .../FunctionInvokingChatClient.cs | 408 +--- .../Common/FunctionInvocationHelpers.cs | 68 + .../Common/FunctionInvocationLogger.cs | 55 + .../Common/FunctionInvocationProcessor.cs | 259 +++ .../Common/FunctionInvocationStatus.cs | 17 + .../OpenTelemetryConsts.cs | 61 + .../AnonymousDelegatingRealtimeSession.cs | 44 + .../FunctionInvokingRealtimeSession.cs | 484 +++++ ...nvokingRealtimeSessionBuilderExtensions.cs | 43 + .../Realtime/LoggingRealtimeSession.cs | 305 +++ ...LoggingRealtimeSessionBuilderExtensions.cs | 57 + .../Realtime/OpenTelemetryRealtimeSession.cs | 1174 ++++++++++ ...lemetryRealtimeSessionBuilderExtensions.cs | 78 + .../Realtime/RealtimeSessionBuilder.cs | 112 + ...SessionBuilderRealtimeSessionExtensions.cs | 28 + .../Realtime/RealtimeSessionExtensions.cs | 30 + .../Realtime/RealtimeAudioFormatTests.cs | 44 + .../Realtime/RealtimeClientMessageTests.cs | 198 ++ .../Realtime/RealtimeContentItemTests.cs | 63 + .../Realtime/RealtimeServerMessageTests.cs | 266 +++ .../Realtime/RealtimeSessionOptionsTests.cs | 131 ++ .../TestRealtimeSession.cs | 70 + .../OpenAIRealtimeClientTests.cs | 82 + ...OpenAIRealtimeSessionSerializationTests.cs | 974 +++++++++ .../OpenAIRealtimeSessionTests.cs | 101 + .../Microsoft.Extensions.AI.Tests.csproj | 1 + .../DelegatingRealtimeSessionTests.cs | 245 +++ .../FunctionInvokingRealtimeSessionTests.cs | 686 ++++++ .../Realtime/LoggingRealtimeSessionTests.cs | 516 +++++ .../OpenTelemetryRealtimeSessionTests.cs | 1343 ++++++++++++ .../Realtime/RealtimeSessionBuilderTests.cs | 227 ++ .../RealtimeSessionExtensionsTests.cs | 50 + 63 files changed, 11231 insertions(+), 319 deletions(-) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClient.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/NoiseReductionOptions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionKind.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/TranscriptionOptions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/ToolChoiceMode.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationHelpers.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationLogger.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationProcessor.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationStatus.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeSession.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSessionBuilderExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSessionBuilderExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSessionBuilderExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilder.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilderRealtimeSessionExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionExtensions.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeAudioFormatTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeContentItemTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionSerializationTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionExtensionsTests.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs new file mode 100644 index 00000000000..5ba84d35734 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs @@ -0,0 +1,79 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Provides an optional base class for an that passes through calls to another instance. +/// +/// +/// This is recommended as a base type when building sessions that can be chained around an underlying . +/// The default implementation simply passes each call to the inner session instance. +/// +[Experimental("MEAI001")] +public class DelegatingRealtimeSession : IRealtimeSession +{ + /// + /// Initializes a new instance of the class. + /// + /// The wrapped session instance. + /// is . + protected DelegatingRealtimeSession(IRealtimeSession innerSession) + { + InnerSession = Throw.IfNull(innerSession); + } + + /// + public void Dispose() + { + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + /// Gets the inner . + protected IRealtimeSession InnerSession { get; } + + /// + public virtual RealtimeSessionOptions? Options => InnerSession.Options; + + /// + public virtual Task InjectClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => + InnerSession.InjectClientMessageAsync(message, cancellationToken); + + /// + public virtual Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancellationToken = default) => + InnerSession.UpdateAsync(options, cancellationToken); + + /// + public virtual IAsyncEnumerable GetStreamingResponseAsync( + IAsyncEnumerable updates, CancellationToken cancellationToken = default) => + InnerSession.GetStreamingResponseAsync(updates, cancellationToken); + + /// + public virtual object? GetService(Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(serviceType); + + // If the key is non-null, we don't know what it means so pass through to the inner service. + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + InnerSession.GetService(serviceType, serviceKey); + } + + /// Provides a mechanism for releasing unmanaged resources. + /// if being called from ; otherwise, . + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + InnerSession.Dispose(); + } + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClient.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClient.cs new file mode 100644 index 00000000000..72ddcc41b77 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClient.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.AI; + +/// Represents a real-time client. +/// This interface provides methods to create and manage real-time sessions. +[Experimental("MEAI001")] +public interface IRealtimeClient : IDisposable +{ + /// Creates a new real-time session with the specified options. + /// The session options. + /// A token to cancel the operation. + /// The created real-time session. + Task CreateSessionAsync(RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default); + + /// Asks the for an object of the specified type . + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly typed services that might be provided by the , + /// including itself or any services it might be wrapping. + /// + object? GetService(Type serviceType, object? serviceKey = null); +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs new file mode 100644 index 00000000000..b813a681d00 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs @@ -0,0 +1,59 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.AI; + +/// Represents a real-time session. +/// This interface provides methods to manage a real-time session and to interact with the real-time model. +[Experimental("MEAI001")] +public interface IRealtimeSession : IDisposable +{ + /// Updates the session with new options. + /// The new session options. + /// A token to cancel the operation. + /// A task that represents the asynchronous update operation. + Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancellationToken = default); + + /// + /// Gets the current session options. + /// + RealtimeSessionOptions? Options { get; } + + /// + /// Injects a client message into the session. + /// + /// The client message to inject. + /// A token to cancel the operation. + /// A task that represents the asynchronous injection operation. + /// + /// This method allows for the injection of client messages into the session at any time, which can be used to influence the session's behavior or state. + /// + Task InjectClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default); + + /// Sends real-time messages and streams the response. + /// The sequence of real-time messages to send. + /// A token to cancel the operation. + /// The response messages generated by the session. + /// + /// This method cannot be called multiple times concurrently on the same session instance. + /// + IAsyncEnumerable GetStreamingResponseAsync( + IAsyncEnumerable updates, CancellationToken cancellationToken = default); + + /// Asks the for an object of the specified type . + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly typed services that might be provided by the , + /// including itself or any services it might be wrapping. + /// + object? GetService(Type serviceType, object? serviceKey = null); +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/NoiseReductionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/NoiseReductionOptions.cs new file mode 100644 index 00000000000..856947d0a4b --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/NoiseReductionOptions.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents options for configuring a real-time session. +/// +[Experimental("MEAI001")] +public enum NoiseReductionOptions +{ + /// + /// No noise reduction applied. + /// + None, + + /// + /// for close-talking microphones. + /// + NearField, + + /// + /// For far-field microphones. + /// + FarField +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs new file mode 100644 index 00000000000..034ce64443c --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents options for configuring real-time audio. +/// +[Experimental("MEAI001")] +public class RealtimeAudioFormat +{ + /// + /// Initializes a new instance of the class. + /// + public RealtimeAudioFormat(string type, int sampleRate) + { + Type = type; + SampleRate = sampleRate; + } + + /// + /// Gets or sets the type of audio. For example, "audio/pcm". + /// + public string Type { get; set; } + + /// + /// Gets or sets the sample rate of the audio in Hertz. + /// + /// + /// When constructed via , this property is always set. + /// The nullable type allows deserialized instances to omit the sample rate when the server does not provide one. + /// + public int? SampleRate { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs new file mode 100644 index 00000000000..623eb6d3a2f --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time message for creating a conversation item. +/// +[Experimental("MEAI001")] +public class RealtimeClientConversationItemCreateMessage : RealtimeClientMessage +{ + /// + /// Initializes a new instance of the class. + /// + /// The conversation item to create. + /// The optional ID of the previous conversation item to insert the new one after. + public RealtimeClientConversationItemCreateMessage(RealtimeContentItem item, string? previousId = null) + { + PreviousId = previousId; + Item = item; + } + + /// + /// Gets or sets the optional previous conversation item ID. + /// If not set, the new item will be appended to the end of the conversation. + /// + public string? PreviousId { get; set; } + + /// + /// Gets or sets the conversation item to create. + /// + public RealtimeContentItem Item { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs new file mode 100644 index 00000000000..1c797bacac8 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time message for appending audio buffer input. +/// +[Experimental("MEAI001")] + +public class RealtimeClientInputAudioBufferAppendMessage : RealtimeClientMessage +{ + /// + /// Initializes a new instance of the class. + /// + /// The data content containing the audio buffer data to append. + public RealtimeClientInputAudioBufferAppendMessage(DataContent audioContent) + { + Content = audioContent; + } + + /// + /// Gets or sets the audio content to append to the model audio buffer. + /// + /// + /// The content should include the audio buffer data that needs to be appended to the input audio buffer. + /// + public DataContent Content { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs new file mode 100644 index 00000000000..e2bc048ff9e --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time message for committing audio buffer input. +/// +[Experimental("MEAI001")] + +public class RealtimeClientInputAudioBufferCommitMessage : RealtimeClientMessage +{ + /// + /// Initializes a new instance of the class. + /// + public RealtimeClientInputAudioBufferCommitMessage() + { + } +} + diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientMessage.cs new file mode 100644 index 00000000000..c0413278264 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientMessage.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time message the client sends to the model. +/// +[Experimental("MEAI001")] +public class RealtimeClientMessage +{ + /// + /// Gets or sets the optional event ID associated with the message. + /// This can be used for tracking and correlation purposes. + /// + public string? EventId { get; set; } + + /// + /// Gets or sets the raw representation of the message. + /// This can be used to send the raw data to the model. + /// + /// + /// The raw representation is typically used for custom or unsupported message types. + /// For example, the model may accept a JSON serialized message. + /// + public object? RawRepresentation { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs new file mode 100644 index 00000000000..6f2e850d052 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs @@ -0,0 +1,86 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time message for creating a response item. +/// +[Experimental("MEAI001")] +public class RealtimeClientResponseCreateMessage : RealtimeClientMessage +{ + /// + /// Initializes a new instance of the class. + /// + public RealtimeClientResponseCreateMessage() + { + } + + /// + /// Gets or sets the list of the conversation items to create a response for. + /// + public IList? Items { get; set; } + + /// + /// Gets or sets the output audio options for the response. If null, the default conversation audio options will be used. + /// + public RealtimeAudioFormat? OutputAudioOptions { get; set; } + + /// + /// Gets or sets the voice of the output audio. + /// + public string? OutputVoice { get; set; } + + /// + /// Gets or sets a value indicating whether the response should be excluded from the conversation history. + /// + public bool ExcludeFromConversation { get; set; } + + /// + /// Gets or sets the instructions allows the client to guide the model on desired responses. + /// If null, the default conversation instructions will be used. + /// + public string? Instructions { get; set; } + + /// + /// Gets or sets the maximum number of output tokens for the response. + /// + public int? MaxOutputTokens { get; set; } + + /// + /// Gets or sets additional metadata for the message. + /// + public AdditionalPropertiesDictionary? Metadata { get; set; } + + /// + /// Gets or sets the output modalities for the response. like "text", "audio". + /// If null, then default conversation modalities will be used. + /// + public IList? OutputModalities { get; set; } + + /// + /// Gets or sets the tool choice mode for the response. + /// + /// + /// If AIFunction or HostedMcpServerTool is specified, this value will be ignored. + /// + public ToolChoiceMode? ToolChoiceMode { get; set; } + + /// + /// Gets or sets the AI function to use for the response. + /// + public AIFunction? AIFunction { get; set; } + + /// + /// Gets or sets the hosted MCP server tool configuration for the response. + /// + public HostedMcpServerTool? HostedMcpServerTool { get; set; } + + /// + /// Gets or sets the AI tools available for generating the response. + /// + public IList? Tools { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs new file mode 100644 index 00000000000..7a3713750fe --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time conversation item. +/// +/// +/// This class is used to encapsulate the details of a real-time item that can be inserted into a conversation, +/// or sent as part of a real-time response creation process. +/// +[Experimental("MEAI001")] +public class RealtimeContentItem +{ + /// + /// Initializes a new instance of the class. + /// + /// The ID of the conversation item. + /// The role of the conversation item. + /// The contents of the conversation item. + public RealtimeContentItem(IList contents, string? id = null, ChatRole? role = null) + { + Id = id; + Role = role; + Contents = contents; + } + + /// + /// Gets or sets the ID of the conversation item. + /// + /// + /// This ID can be null in case passing Function or MCP content where the ID is not required. + /// The Id only needed of having contents representing a user, system, or assistant message with contents like text, audio, image or similar. + /// + public string? Id { get; set; } + + /// + /// Gets or sets the role of the conversation item. + /// + /// + /// The role not used in case of Function or MCP content. + /// The role only needed of having contents representing a user, system, or assistant message with contents like text, audio, image or similar. + /// + public ChatRole? Role { get; set; } + + /// + /// Gets or sets the content of the conversation item. + /// + public IList Contents { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs new file mode 100644 index 00000000000..b75e91793e1 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs @@ -0,0 +1,40 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time server error message. +/// +/// +/// Used with the . +/// +[Experimental("MEAI001")] +public class RealtimeServerErrorMessage : RealtimeServerMessage +{ + /// + /// Initializes a new instance of the class. + /// + public RealtimeServerErrorMessage() + { + Type = RealtimeServerMessageType.Error; + } + + /// + /// Gets or sets the error content associated with the error message. + /// + public ErrorContent? Error { get; set; } + + /// + /// Gets or sets an optional event ID caused the error. + /// + public string? ErrorEventId { get; set; } + + /// + /// Gets or sets an optional parameter providing additional context about the error. + /// + public string? Parameter { get; set; } + +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs new file mode 100644 index 00000000000..6e53133a41e --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time server message for input audio transcription. +/// +/// +/// Used when having InputAudioTranscriptionCompleted, InputAudioTranscriptionDelta, or InputAudioTranscriptionFailed response types. +/// +[Experimental("MEAI001")] +public class RealtimeServerInputAudioTranscriptionMessage : RealtimeServerMessage +{ + /// + /// Initializes a new instance of the class. + /// + /// The type of the real-time server response. + /// + /// The parameter should be InputAudioTranscriptionCompleted, InputAudioTranscriptionDelta, or InputAudioTranscriptionFailed. + /// + public RealtimeServerInputAudioTranscriptionMessage(RealtimeServerMessageType type) + { + Type = type; + } + + /// + /// Gets or sets the index of the content part containing the audio. + /// + public int? ContentIndex { get; set; } + + /// + /// Gets or sets the ID of the item containing the audio that is being transcribed. + /// + public string? ItemId { get; set; } + + /// + /// Gets or sets the transcription text of the audio. + /// + public string? Transcription { get; set; } + + /// + /// Gets or sets the usage details for the transcription. + /// + public UsageDetails? Usage { get; set; } + + /// + /// Gets or sets the error content if an error occurred during transcription. + /// + public ErrorContent? Error { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessage.cs new file mode 100644 index 00000000000..a6eb9e03d70 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessage.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time server response message. +/// +[Experimental("MEAI001")] +public class RealtimeServerMessage +{ + /// + /// Gets or sets the type of the real-time response. + /// + public RealtimeServerMessageType Type { get; set; } + + /// + /// Gets or sets the optional event ID associated with the response. + /// This can be used for tracking and correlation purposes. + /// + public string? EventId { get; set; } + + /// + /// Gets or sets the raw representation of the response. + /// This can be used to hold the original data structure received from the model. + /// + /// + /// The raw representation is typically used for custom or unsupported message types. + /// For example, the model may accept a JSON serialized server message. + /// + public object? RawRepresentation { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs new file mode 100644 index 00000000000..bbe7e78f08b --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs @@ -0,0 +1,148 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents the type of a real-time response. +/// This is used to identify the response type being received from the model. +/// +[Experimental("MEAI001")] +public enum RealtimeServerMessageType +{ + /// + /// Indicates that the response contains only raw content. + /// + /// + /// This response type is to support extensibility for supporting custom content types not natively supported by the SDK. + /// + RawContentOnly, + + /// + /// Indicates the output of audio transcription for user audio written to the user audio buffer. + /// + /// + /// The type is used with this response type. + /// + InputAudioTranscriptionCompleted, + + /// + /// Indicates the text value of an input audio transcription content part is updated with incremental transcription results. + /// + /// + /// The type is used with this response type. + /// + InputAudioTranscriptionDelta, + + /// + /// Indicates that the audio transcription for user audio written to the user audio buffer has failed. + /// + /// + /// The type is used with this response type. + /// + InputAudioTranscriptionFailed, + + /// + /// Indicates the output text update with incremental results response. + /// + /// + /// The type is used with this response type. + /// + OutputTextDelta, + + /// + /// Indicates the output text is complete. + /// + /// + /// The type is used with this response type. + /// + OutputTextDone, + + /// + /// Indicates the model-generated transcription of audio output updated. + /// + /// + /// The type is used with this response type. + /// + OutputAudioTranscriptionDelta, + + /// + /// Indicates the model-generated transcription of audio output is done streaming. + /// + /// + /// The type is used with this response type. + /// + OutputAudioTranscriptionDone, + + /// + /// Indicates the audio output updated. + /// + /// + /// The type is used with this response type. + /// + OutputAudioDelta, + + /// + /// Indicates the audio output is done streaming. + /// + /// + /// The type is used with this response type. + /// + OutputAudioDone, + + /// + /// Indicates the response has been created. + /// + /// + /// The type is used with this response type. + /// + ResponseDone, + + /// + /// Indicates the response has been created. + /// + /// + /// The type is used with this response type. + /// + ResponseCreated, + + /// + /// Indicates an error occurred while processing the request. + /// + /// + /// The type is used with this response type. + /// + Error, + + /// + /// Indicates that an MCP tool call is in progress. + /// + McpCallInProgress, + + /// + /// Indicates that an MCP tool call has completed. + /// + McpCallCompleted, + + /// + /// Indicates that an MCP tool call has failed. + /// + McpCallFailed, + + /// + /// Indicates that listing MCP tools is in progress. + /// + McpListToolsInProgress, + + /// + /// Indicates that listing MCP tools has completed. + /// + McpListToolsCompleted, + + /// + /// Indicates that listing MCP tools has failed. + /// + McpListToolsFailed, +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs new file mode 100644 index 00000000000..b44b0e969cc --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs @@ -0,0 +1,56 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time server message for output text and audio. +/// +[Experimental("MEAI001")] +public class RealtimeServerOutputTextAudioMessage : RealtimeServerMessage +{ + /// + /// Initializes a new instance of the class for handling output text delta responses. + /// + /// The type of the real-time server response. + /// + /// The should be , , + /// , , + /// , or . + /// + public RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType type) + { + Type = type; + } + + /// + /// Gets or sets the index of the content part whose text has been updated. + /// + public int? ContentIndex { get; set; } + + /// + /// Gets or sets the text or audio delta, or the final text or audio once the output is complete. + /// + /// + /// if dealing with audio content, this property may contain Base64-encoded audio data. + /// With , usually will have null Text value. + /// + public string? Text { get; set; } + + /// + /// Gets or sets the ID of the item containing the content part whose text has been updated. + /// + public string? ItemId { get; set; } + + /// + /// Gets or sets the index of the output item in the response. + /// + public int? OutputIndex { get; set; } + + /// + /// Gets or sets the ID of the response. + /// + public string? ResponseId { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs new file mode 100644 index 00000000000..d4a94875b8b --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs @@ -0,0 +1,85 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time message for creating a response item. +/// +/// +/// Used with the and messages. +/// +[Experimental("MEAI001")] +public class RealtimeServerResponseCreatedMessage : RealtimeServerMessage +{ + /// + /// Initializes a new instance of the class. + /// + /// + /// The should be or . + /// + public RealtimeServerResponseCreatedMessage(RealtimeServerMessageType type) + { + Type = type; + } + + /// + /// Gets or sets the output audio options for the response. If null, the default conversation audio options will be used. + /// + public RealtimeAudioFormat? OutputAudioOptions { get; set; } + + /// + /// Gets or sets the voice of the output audio. + /// + public string? OutputVoice { get; set; } + + /// + /// Gets or sets the conversation ID associated with the response. + /// + public string? ConversationId { get; set; } + + /// + /// Gets or sets the unique response ID. + /// + public string? ResponseId { get; set; } + + /// + /// Gets or sets the maximum number of output tokens for the response. + /// If 0, the service will apply its own limit. + /// + public int? MaxOutputTokens { get; set; } + + /// + /// Gets or sets additional metadata for the message. + /// + public AdditionalPropertiesDictionary? Metadata { get; set; } + + /// + /// Gets or sets the list of the conversation items included in the response. + /// + public IList? Items { get; set; } + + /// + /// Gets or sets the output modalities for the response. like "text", "audio". + /// If null, then default conversation modalities will be used. + /// + public IList? OutputModalities { get; set; } + + /// + /// Gets or sets the status of the response. + /// + public string? Status { get; set; } + + /// + /// Gets or sets the error content of the response, if any. + /// + public ErrorContent? Error { get; set; } + + /// + /// Gets or sets the usage details for the response. + /// + public UsageDetails? Usage { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs new file mode 100644 index 00000000000..f6007e6948d --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a real-time message representing a new output item added or created during response generation. +/// +/// +/// Used with the and messages. +/// +[Experimental("MEAI001")] +public class RealtimeServerResponseOutputItemMessage : RealtimeServerMessage +{ + /// + /// Initializes a new instance of the class. + /// + /// + /// The should be or . + /// + public RealtimeServerResponseOutputItemMessage(RealtimeServerMessageType type) + { + Type = type; + } + + /// + /// Gets or sets the unique response ID. + /// + public string? ResponseId { get; set; } + + /// + /// Gets or sets the unique output index. + /// + public int? OutputIndex { get; set; } + + /// + /// Gets or sets the conversation item included in the response. + /// + public RealtimeContentItem? Item { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionKind.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionKind.cs new file mode 100644 index 00000000000..05be70bffda --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionKind.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents options for configuring a real-time session. +/// +[Experimental("MEAI001")] +public enum RealtimeSessionKind +{ + /// + /// Represent a realtime sessions which process audio, text, or other media in real-time. + /// + Realtime, + + /// + /// Represent transcription only session. + /// + Transcription +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs new file mode 100644 index 00000000000..5b2ede4a7f2 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs @@ -0,0 +1,138 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// Represents options for configuring a real-time session. +[Experimental("MEAI001")] +public class RealtimeSessionOptions +{ + /// + /// Gets or sets the session kind. + /// + /// + /// If set to , most of the sessions properties will not apply to the session. Only InputAudioFormat, NoiseReductionOptions, TranscriptionOptions, and VoiceActivityDetection will be used. + /// + public RealtimeSessionKind SessionKind { get; set; } = RealtimeSessionKind.Realtime; + + /// + /// Gets or sets the model name to use for the session. + /// + public string? Model { get; set; } + + /// + /// Gets or sets the input audio format for the session. + /// + public RealtimeAudioFormat? InputAudioFormat { get; set; } + + /// + /// Gets or sets the noise reduction options for the session. + /// + public NoiseReductionOptions? NoiseReductionOptions { get; set; } + + /// + /// Gets or sets the transcription options for the session. + /// + public TranscriptionOptions? TranscriptionOptions { get; set; } + + /// + /// Gets or sets the voice activity detection options for the session. + /// + public VoiceActivityDetection? VoiceActivityDetection { get; set; } + + /// + /// Gets or sets the output audio format for the session. + /// + public RealtimeAudioFormat? OutputAudioFormat { get; set; } + + /// + /// Gets or sets the output voice speed for the session. + /// + /// + /// The default value is 1.0, which represents normal speed. + /// + public double VoiceSpeed { get; set; } = 1.0; + + /// + /// Gets or sets the output voice for the session. + /// + public string? Voice { get; set; } + + /// + /// Gets or sets the default system instructions for the session. + /// + public string? Instructions { get; set; } + + /// + /// Gets or sets the maximum number of response tokens for the session. + /// + public int? MaxOutputTokens { get; set; } + + /// + /// Gets or sets the output modalities for the response. like "text", "audio". + /// If null, then default conversation modalities will be used. + /// + public IList? OutputModalities { get; set; } + + /// + /// Gets or sets the tool choice mode for the response. + /// + /// + /// If FunctionToolName or McpToolName is specified, this value will be ignored. + /// + public ToolChoiceMode? ToolChoiceMode { get; set; } + + /// + /// Gets or sets the AI function to use for the response. + /// + /// + /// If specified, the ToolChoiceMode will be ignored. + /// + public AIFunction? AIFunction { get; set; } + + /// + /// Gets or sets the name of the MCP tool to use for the response. + /// + /// + /// If specified, the ToolChoiceMode will be ignored. + /// + public HostedMcpServerTool? HostedMcpServerTool { get; set; } + + /// + /// Gets or sets the AI tools available for generating the response. + /// + public IList? Tools { get; set; } + + /// + /// Gets or sets a value indicating whether to enable automatic tracing for the session. + /// if enabled, will create a trace for the session with default values for the workflow name, group id, and metadata. + /// + public bool EnableAutoTracing { get; set; } + + /// + /// Gets or sets the group ID for tracing. + /// + /// + /// This property is only used if is not set to true. + /// + public string? TracingGroupId { get; set; } + + /// + /// Gets or sets the workflow name for tracing. + /// + /// + /// This property is only used if is not set to true. + /// + public string? TracingWorkflowName { get; set; } + + /// + /// Gets or sets arbitrary metadata to attach to this trace to enable filtering. + /// + /// + /// This property is only used if is not set to true. + /// + public object? TracingMetadata { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs new file mode 100644 index 00000000000..74e0d699fe9 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents options for configuring server voice activity detection in a real-time session. +/// +[Experimental("MEAI001")] +public class SemanticVoiceActivityDetection : VoiceActivityDetection +{ + /// + /// Gets or sets the eagerness level for semantic voice activity detection. + /// + /// + /// Examples of the values are "low", "medium", "high", and "auto". + /// + public string Eagerness { get; set; } = "auto"; +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs new file mode 100644 index 00000000000..2a0f67970ab --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents options for configuring server voice activity detection in a real-time session. +/// +[Experimental("MEAI001")] +public class ServerVoiceActivityDetection : VoiceActivityDetection +{ + /// + /// Gets or sets the idle timeout in milliseconds to detect the end of speech. + /// + public int IdleTimeoutInMilliseconds { get; set; } + + /// + /// Gets or sets the prefix padding in milliseconds to include before detected speech. + /// + public int PrefixPaddingInMilliseconds { get; set; } = 300; + + /// + /// Gets or sets the silence duration in milliseconds to consider as a pause. + /// + public int SilenceDurationInMilliseconds { get; set; } = 500; + + /// + /// Gets or sets the threshold for voice activity detection. + /// + /// + /// A value between 0.0 and 1.0, where higher values make the detection more sensitive. + /// + public double Threshold { get; set; } = 0.5; +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/TranscriptionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/TranscriptionOptions.cs new file mode 100644 index 00000000000..b0287112a3a --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/TranscriptionOptions.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents options for configuring transcription in a real-time session. +/// +[Experimental("MEAI001")] +public class TranscriptionOptions +{ + /// + /// Initializes a new instance of the class. + /// + public TranscriptionOptions(string language, string model, string? prompt = null) + { + Language = language; + Model = model; + Prompt = prompt; + } + + /// + /// Gets or sets the language for transcription. The input language should be in ISO-639-1 (e.g. en). + /// + public string Language { get; set; } + + /// + /// Gets or sets the model name to use for transcription. + /// + public string Model { get; set; } + + /// + /// Gets or sets an optional prompt to guide the transcription. + /// + public string? Prompt { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs new file mode 100644 index 00000000000..b7e63616460 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents options for configuring voice activity detection in a real-time session. +/// +[Experimental("MEAI001")] +public class VoiceActivityDetection +{ + /// + /// Gets or sets a value indicating whether to create a response when voice activity is detected. + /// + public bool CreateResponse { get; set; } + + /// + /// Gets or sets a value indicating whether to interrupt the response when voice activity is detected. + /// + public bool InterruptResponse { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/ToolChoiceMode.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/ToolChoiceMode.cs new file mode 100644 index 00000000000..6093d3b2add --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/ToolChoiceMode.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Defines modes that controls which if any tool is called by the model. +/// +[Experimental("MEAI001")] +public enum ToolChoiceMode +{ + /// + /// The model will not call any tool and instead generates a message. + /// + None, + + /// + /// The model can pick between generating a message or calling one or more tools. + /// + Auto, + + /// + /// The model must call one or more tools. + /// + Required +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/UsageDetails.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/UsageDetails.cs index b3edbad5e99..a82d1a4f9f1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/UsageDetails.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/UsageDetails.cs @@ -4,6 +4,8 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; @@ -38,6 +40,38 @@ public class UsageDetails /// public long? ReasoningTokenCount { get; set; } + /// Gets or sets the number of audio input tokens used. + /// + /// This property is used only when audio input tokens are involved. + /// + [Experimental("MEAI001")] + [JsonIgnore] + public long? InputAudioTokenCount { get; set; } + + /// Gets or sets the number of text input tokens used. + /// + /// This property is used only when having audio and text tokens. Otherwise InputTokenCount is sufficient. + /// + [Experimental("MEAI001")] + [JsonIgnore] + public long? InputTextTokenCount { get; set; } + + /// Gets or sets the number of audio output tokens used. + /// + /// This property is used only when audio output tokens are involved. + /// + [Experimental("MEAI001")] + [JsonIgnore] + public long? OutputAudioTokenCount { get; set; } + + /// Gets or sets the number of text output tokens used. + /// + /// This property is used only when having audio and text tokens. Otherwise OutputTokenCount is sufficient. + /// + [Experimental("MEAI001")] + [JsonIgnore] + public long? OutputTextTokenCount { get; set; } + /// Gets or sets a dictionary of additional usage counts. /// /// All values set here are assumed to be summable. For example, when middleware makes multiple calls to an underlying @@ -57,6 +91,8 @@ public void Add(UsageDetails usage) TotalTokenCount = NullableSum(TotalTokenCount, usage.TotalTokenCount); CachedInputTokenCount = NullableSum(CachedInputTokenCount, usage.CachedInputTokenCount); ReasoningTokenCount = NullableSum(ReasoningTokenCount, usage.ReasoningTokenCount); + InputAudioTokenCount = NullableSum(InputAudioTokenCount, usage.InputAudioTokenCount); + InputTextTokenCount = NullableSum(InputTextTokenCount, usage.InputTextTokenCount); if (usage.AdditionalCounts is { } countsToAdd) { @@ -109,6 +145,16 @@ internal string DebuggerDisplay parts.Add($"{nameof(ReasoningTokenCount)} = {reasoning}"); } + if (InputAudioTokenCount is { } inputAudio) + { + parts.Add($"{nameof(InputAudioTokenCount)} = {inputAudio}"); + } + + if (InputTextTokenCount is { } inputText) + { + parts.Add($"{nameof(InputTextTokenCount)} = {inputText}"); + } + if (AdditionalCounts is { } additionalCounts) { foreach (var entry in additionalCounts) diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Microsoft.Extensions.AI.Evaluation.Reporting.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Microsoft.Extensions.AI.Evaluation.Reporting.csproj index 8ee31bc2b1a..8a960fc4df1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Microsoft.Extensions.AI.Evaluation.Reporting.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Microsoft.Extensions.AI.Evaluation.Reporting.csproj @@ -1,6 +1,6 @@  - + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + true + + \ No newline at end of file From a6b4ccdd168c513acd7497323af1a2b951bfb349 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Fri, 13 Feb 2026 16:26:20 -0800 Subject: [PATCH 17/92] Replace string Eagerness with SemanticEagerness struct (ChatRole pattern) --- .../Realtime/SemanticEagerness.cs | 95 +++++++++++++++++++ .../SemanticVoiceActivityDetection.cs | 5 +- .../OpenAIRealtimeSession.cs | 4 +- ...OpenAIRealtimeSessionSerializationTests.cs | 2 +- 4 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticEagerness.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticEagerness.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticEagerness.cs new file mode 100644 index 00000000000..3d6cabe4950 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticEagerness.cs @@ -0,0 +1,95 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.ComponentModel; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents the eagerness level for semantic voice activity detection. +/// +[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] +[JsonConverter(typeof(Converter))] +[DebuggerDisplay("{Value,nq}")] +public readonly struct SemanticEagerness : IEquatable +{ + /// Gets a value representing low eagerness. + public static SemanticEagerness Low { get; } = new("low"); + + /// Gets a value representing medium eagerness. + public static SemanticEagerness Medium { get; } = new("medium"); + + /// Gets a value representing high eagerness. + public static SemanticEagerness High { get; } = new("high"); + + /// Gets a value representing automatic eagerness detection. + public static SemanticEagerness Auto { get; } = new("auto"); + + /// + /// Gets the value associated with this . + /// + public string Value { get; } + + /// + /// Initializes a new instance of the struct with the provided value. + /// + /// The value to associate with this . + [JsonConstructor] + public SemanticEagerness(string value) + { + Value = Throw.IfNullOrWhitespace(value); + } + + /// + /// Returns a value indicating whether two instances are equivalent, as determined by a + /// case-insensitive comparison of their values. + /// + public static bool operator ==(SemanticEagerness left, SemanticEagerness right) + { + return left.Equals(right); + } + + /// + /// Returns a value indicating whether two instances are not equivalent, as determined by a + /// case-insensitive comparison of their values. + /// + public static bool operator !=(SemanticEagerness left, SemanticEagerness right) + { + return !(left == right); + } + + /// + public override bool Equals([NotNullWhen(true)] object? obj) + => obj is SemanticEagerness other && Equals(other); + + /// + public bool Equals(SemanticEagerness other) + => string.Equals(Value, other.Value, StringComparison.OrdinalIgnoreCase); + + /// + public override int GetHashCode() + => StringComparer.OrdinalIgnoreCase.GetHashCode(Value); + + /// + public override string ToString() => Value; + + /// Provides a for serializing instances. + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class Converter : JsonConverter + { + /// + public override SemanticEagerness Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + new(reader.GetString()!); + + /// + public override void Write(Utf8JsonWriter writer, SemanticEagerness value, JsonSerializerOptions options) => + Throw.IfNull(writer).WriteStringValue(value.Value); + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs index 3a406cdda27..12f995d5b42 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs @@ -15,8 +15,5 @@ public class SemanticVoiceActivityDetection : VoiceActivityDetection /// /// Gets or sets the eagerness level for semantic voice activity detection. /// - /// - /// Examples of the values are "low", "medium", "high", and "auto". - /// - public string Eagerness { get; set; } = "auto"; + public SemanticEagerness Eagerness { get; set; } = SemanticEagerness.Auto; } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 09be4c82fea..e2716c2262c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -190,7 +190,7 @@ public async Task UpdateAsync(RealtimeSessionOptions options, CancellationToken ["type"] = "semantic_vad", ["create_response"] = semanticVad.CreateResponse, ["interrupt_response"] = semanticVad.InterruptResponse, - ["eagerness"] = semanticVad.Eagerness, + ["eagerness"] = semanticVad.Eagerness.Value, }; } @@ -1505,7 +1505,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (turnDetectionElement.TryGetProperty("eagerness", out var eagernessElement) && eagernessElement.GetString() is string eagerness) { - semanticVad.Eagerness = eagerness; + semanticVad.Eagerness = new SemanticEagerness(eagerness); } options.VoiceActivityDetection = semanticVad; diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionSerializationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionSerializationTests.cs index 636652900a5..da5eb80bfc7 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionSerializationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionSerializationTests.cs @@ -270,7 +270,7 @@ public async Task ProcessServerEvent_SessionUpdated_WithSemanticVad() var semanticVad = Assert.IsType(options.VoiceActivityDetection); Assert.False(semanticVad.CreateResponse); Assert.True(semanticVad.InterruptResponse); - Assert.Equal("high", semanticVad.Eagerness); + Assert.Equal(SemanticEagerness.High, semanticVad.Eagerness); } [Fact] From 03d76488bdd18d9fa1e7686456f52b2bcee64c12 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 09:27:25 -0800 Subject: [PATCH 18/92] Remove InternalsVisibleToTest from Microsoft.Extensions.AI.OpenAI - Make OpenAIRealtimeSession constructor and ConnectAsync public - Remove InternalsVisibleToTest from csproj - Remove OpenAIRealtimeSessionSerializationTests (depended on internal ConnectWithWebSocketAsync) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Microsoft.Extensions.AI.OpenAI.csproj | 4 - .../OpenAIRealtimeSession.cs | 4 +- ...OpenAIRealtimeSessionSerializationTests.cs | 974 ------------------ 3 files changed, 2 insertions(+), 980 deletions(-) delete mode 100644 test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionSerializationTests.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj b/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj index 656f149df95..89ff9b90c29 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj @@ -31,10 +31,6 @@ true - - - - diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index e2716c2262c..cc078142c6a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -62,7 +62,7 @@ public sealed class OpenAIRealtimeSession : IRealtimeSession /// Initializes a new instance of the class. /// The API key used for authentication. /// The model to use for the session. - internal OpenAIRealtimeSession(string apiKey, string model) + public OpenAIRealtimeSession(string apiKey, string model) { _apiKey = apiKey; _model = model; @@ -72,7 +72,7 @@ internal OpenAIRealtimeSession(string apiKey, string model) /// Connects the WebSocket to the OpenAI Realtime API. /// The to monitor for cancellation requests. /// if the connection succeeded; otherwise, . - internal async Task ConnectAsync(CancellationToken cancellationToken = default) + public async Task ConnectAsync(CancellationToken cancellationToken = default) { try { diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionSerializationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionSerializationTests.cs deleted file mode 100644 index da5eb80bfc7..00000000000 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionSerializationTests.cs +++ /dev/null @@ -1,974 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Net.WebSockets; -using System.Runtime.CompilerServices; -using System.Text; -using System.Text.Json; -using System.Text.Json.Nodes; -using System.Threading; -using System.Threading.Channels; -using System.Threading.Tasks; -using Xunit; - -#pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. -#pragma warning disable SA1204 // Static elements should appear before instance elements -#pragma warning disable S103 // Lines should not be too long - -namespace Microsoft.Extensions.AI; - -/// -/// Tests for the JSON serialization and deserialization logic used by . -/// Uses a channel-backed WebSocket pair to exercise message processing without a real network connection. -/// -public class OpenAIRealtimeSessionSerializationTests : IAsyncLifetime -{ - private readonly ChannelWebSocket _serverWebSocket; - private readonly ChannelWebSocket _clientWebSocket; - private readonly OpenAIRealtimeSession _session; - private readonly CancellationTokenSource _cts = new(); - - public OpenAIRealtimeSessionSerializationTests() - { - var clientToServer = Channel.CreateUnbounded<(byte[] data, WebSocketMessageType messageType, bool endOfMessage)>(); - var serverToClient = Channel.CreateUnbounded<(byte[] data, WebSocketMessageType messageType, bool endOfMessage)>(); - - _clientWebSocket = new ChannelWebSocket(serverToClient.Reader, clientToServer.Writer); - _serverWebSocket = new ChannelWebSocket(clientToServer.Reader, serverToClient.Writer); - _session = new OpenAIRealtimeSession("test-key", "test-model"); - } - - public async Task InitializeAsync() - { - await _session.ConnectWithWebSocketAsync(_clientWebSocket, _cts.Token); - } - - public Task DisposeAsync() - { - _cts.Cancel(); - _session.Dispose(); - _clientWebSocket.Dispose(); - _serverWebSocket.Dispose(); - _cts.Dispose(); - return Task.CompletedTask; - } - - [Fact] - public async Task ProcessServerEvent_ErrorMessage_ParsedCorrectly() - { - var errorJson = """{"type":"error","event_id":"evt_001","error":{"message":"Something went wrong","code":"invalid_request","param":"model"}}"""; - - await SendServerMessageAsync(errorJson); - var msg = await ReadNextServerMessageAsync(); - - var errorMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.Error, errorMsg.Type); - Assert.Equal("evt_001", errorMsg.EventId); - Assert.Equal("Something went wrong", errorMsg.Error?.Message); - Assert.Equal("invalid_request", errorMsg.Error?.ErrorCode); - Assert.Equal("model", errorMsg.Parameter); - } - - [Fact] - public async Task ProcessServerEvent_InputAudioTranscriptionDelta_ParsedCorrectly() - { - var json = """{"type":"conversation.item.input_audio_transcription.delta","event_id":"evt_002","item_id":"item_001","content_index":0,"delta":"Hello world"}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var transcription = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.InputAudioTranscriptionDelta, transcription.Type); - Assert.Equal("evt_002", transcription.EventId); - Assert.Equal("item_001", transcription.ItemId); - Assert.Equal(0, transcription.ContentIndex); - Assert.Equal("Hello world", transcription.Transcription); - } - - [Fact] - public async Task ProcessServerEvent_InputAudioTranscriptionCompleted_UsesTranscriptField() - { - var json = """{"type":"conversation.item.input_audio_transcription.completed","event_id":"evt_003","item_id":"item_002","content_index":1,"transcript":"The full transcription"}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var transcription = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.InputAudioTranscriptionCompleted, transcription.Type); - Assert.Equal("The full transcription", transcription.Transcription); - } - - [Fact] - public async Task ProcessServerEvent_InputAudioTranscriptionFailed_ParsedCorrectly() - { - var json = """{"type":"conversation.item.input_audio_transcription.failed","event_id":"evt_004","item_id":"item_003","error":{"message":"Transcription failed","code":"transcription_error","param":"audio"}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var transcription = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.InputAudioTranscriptionFailed, transcription.Type); - Assert.NotNull(transcription.Error); - Assert.Equal("Transcription failed", transcription.Error.Message); - Assert.Equal("transcription_error", transcription.Error.ErrorCode); - } - - [Fact] - public async Task ProcessServerEvent_OutputAudioTranscriptDelta_ParsedCorrectly() - { - var json = """{"type":"response.output_audio_transcript.delta","event_id":"evt_005","response_id":"resp_001","item_id":"item_004","output_index":0,"content_index":0,"delta":"Hello"}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.OutputAudioTranscriptionDelta, outputMsg.Type); - Assert.Equal("evt_005", outputMsg.EventId); - Assert.Equal("resp_001", outputMsg.ResponseId); - Assert.Equal("item_004", outputMsg.ItemId); - Assert.Equal(0, outputMsg.OutputIndex); - Assert.Equal(0, outputMsg.ContentIndex); - Assert.Equal("Hello", outputMsg.Text); - } - - [Fact] - public async Task ProcessServerEvent_OutputAudioDelta_ParsedCorrectly() - { - var json = """{"type":"response.output_audio.delta","event_id":"evt_006","response_id":"resp_002","item_id":"item_005","output_index":0,"content_index":0,"delta":"base64audiodata"}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.OutputAudioDelta, outputMsg.Type); - Assert.Equal("base64audiodata", outputMsg.Text); - } - - [Fact] - public async Task ProcessServerEvent_ResponseCreated_ParsedCorrectly() - { - var json = """{"type":"response.created","event_id":"evt_007","response":{"id":"resp_003","conversation_id":"conv_001","status":"in_progress","max_output_tokens":4096,"output_modalities":["text","audio"],"metadata":{"key1":"value1"},"audio":{"output":{"format":{"type":"audio/pcm","rate":24000},"voice":"alloy"}}}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var responseMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.ResponseCreated, responseMsg.Type); - Assert.Equal("evt_007", responseMsg.EventId); - Assert.Equal("resp_003", responseMsg.ResponseId); - Assert.Equal("conv_001", responseMsg.ConversationId); - Assert.Equal("in_progress", responseMsg.Status); - Assert.Equal(4096, responseMsg.MaxOutputTokens); - Assert.NotNull(responseMsg.OutputModalities); - Assert.Equal(new[] { "text", "audio" }, responseMsg.OutputModalities); - Assert.NotNull(responseMsg.Metadata); - Assert.Equal("value1", responseMsg.Metadata["key1"]); - Assert.NotNull(responseMsg.OutputAudioOptions); - Assert.Equal("audio/pcm", responseMsg.OutputAudioOptions.Type); - Assert.Equal(24000, responseMsg.OutputAudioOptions.SampleRate); - Assert.Equal("alloy", responseMsg.OutputVoice); - } - - [Fact] - public async Task ProcessServerEvent_ResponseDone_WithUsageAndOutput_ParsedCorrectly() - { - var json = """{"type":"response.done","event_id":"evt_008","response":{"id":"resp_004","status":"completed","usage":{"input_tokens":100,"output_tokens":50,"total_tokens":150},"output":[{"type":"message","id":"msg_001","role":"assistant","content":[{"type":"input_text","text":"Hello there!"}]}]}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var responseMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.ResponseDone, responseMsg.Type); - Assert.Equal("resp_004", responseMsg.ResponseId); - Assert.Equal("completed", responseMsg.Status); - Assert.NotNull(responseMsg.Usage); - Assert.Equal(100, responseMsg.Usage.InputTokenCount); - Assert.Equal(50, responseMsg.Usage.OutputTokenCount); - Assert.Equal(150, responseMsg.Usage.TotalTokenCount); - Assert.NotNull(responseMsg.Items); - Assert.Single(responseMsg.Items); - Assert.Equal("msg_001", responseMsg.Items[0].Id); - Assert.Equal(ChatRole.Assistant, responseMsg.Items[0].Role); - var textContent = Assert.IsType(responseMsg.Items[0].Contents[0]); - Assert.Equal("Hello there!", textContent.Text); - } - - [Fact] - public async Task ProcessServerEvent_OutputItemAdded_WithFunctionCall_ParsedCorrectly() - { - var json = """{"type":"response.output_item.added","event_id":"evt_010","response_id":"resp_006","output_index":0,"item":{"type":"function_call","id":"fc_001","name":"get_weather","call_id":"call_001","arguments":"{\"city\":\"Seattle\"}"}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputItemMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.ResponseOutputItemAdded, outputItemMsg.Type); - Assert.Equal("resp_006", outputItemMsg.ResponseId); - Assert.Equal(0, outputItemMsg.OutputIndex); - Assert.NotNull(outputItemMsg.Item); - Assert.Equal("fc_001", outputItemMsg.Item.Id); - var functionCall = Assert.IsType(outputItemMsg.Item.Contents[0]); - Assert.Equal("call_001", functionCall.CallId); - Assert.Equal("get_weather", functionCall.Name); - Assert.NotNull(functionCall.Arguments); - Assert.Equal("Seattle", functionCall.Arguments["city"]?.ToString()); - } - - [Fact] - public async Task ProcessServerEvent_SessionCreated_UpdatesOptions() - { - var json = """{"type":"session.created","event_id":"evt_011","session":{"type":"realtime","model":"gpt-realtime","instructions":"Be helpful","max_output_tokens":2048,"output_modalities":["text"],"audio":{"input":{"format":{"type":"audio/pcm","rate":16000},"noise_reduction":{"type":"near_field"},"transcription":{"language":"en","model":"whisper-1"},"turn_detection":{"type":"server_vad","create_response":true,"interrupt_response":true,"idle_timeout_ms":5000,"prefix_padding_ms":300,"silence_duration_ms":500,"threshold":0.5}},"output":{"format":{"type":"audio/pcm","rate":24000},"speed":1.0,"voice":"alloy"}}}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - Assert.Equal(RealtimeServerMessageType.RawContentOnly, msg.Type); - Assert.NotNull(msg.RawRepresentation); - - var options = _session.Options; - Assert.NotNull(options); - Assert.Equal(RealtimeSessionKind.Realtime, options.SessionKind); - Assert.Equal("gpt-realtime", options.Model); - Assert.Equal("Be helpful", options.Instructions); - Assert.Equal(2048, options.MaxOutputTokens); - Assert.NotNull(options.InputAudioFormat); - Assert.Equal("audio/pcm", options.InputAudioFormat.Type); - Assert.Equal(16000, options.InputAudioFormat.SampleRate); - Assert.Equal(NoiseReductionOptions.NearField, options.NoiseReductionOptions); - Assert.NotNull(options.TranscriptionOptions); - Assert.Equal("en", options.TranscriptionOptions.SpeechLanguage); - Assert.Equal("whisper-1", options.TranscriptionOptions.ModelId); - - var serverVad = Assert.IsType(options.VoiceActivityDetection); - Assert.True(serverVad.CreateResponse); - Assert.True(serverVad.InterruptResponse); - Assert.Equal(5000, serverVad.IdleTimeoutInMilliseconds); - Assert.Equal(300, serverVad.PrefixPaddingInMilliseconds); - Assert.Equal(500, serverVad.SilenceDurationInMilliseconds); - Assert.Equal(0.5, serverVad.Threshold); - - Assert.NotNull(options.OutputAudioFormat); - Assert.Equal("audio/pcm", options.OutputAudioFormat.Type); - Assert.Equal(24000, options.OutputAudioFormat.SampleRate); - Assert.Equal("alloy", options.Voice); - } - - [Fact] - public async Task ProcessServerEvent_SessionUpdated_WithSemanticVad() - { - var json = """{"type":"session.updated","event_id":"evt_012","session":{"type":"transcription","audio":{"input":{"turn_detection":{"type":"semantic_vad","create_response":false,"interrupt_response":true,"eagerness":"high"}}}}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - Assert.Equal(RealtimeServerMessageType.RawContentOnly, msg.Type); - var options = _session.Options; - Assert.NotNull(options); - Assert.Equal(RealtimeSessionKind.Transcription, options.SessionKind); - var semanticVad = Assert.IsType(options.VoiceActivityDetection); - Assert.False(semanticVad.CreateResponse); - Assert.True(semanticVad.InterruptResponse); - Assert.Equal(SemanticEagerness.High, semanticVad.Eagerness); - } - - [Fact] - public async Task ProcessServerEvent_UnknownEventType_ParsedAsRawContentOnly() - { - var json = """{"type":"some.unknown.event","event_id":"evt_013","data":{"key":"value"}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - Assert.Equal(RealtimeServerMessageType.RawContentOnly, msg.Type); - Assert.NotNull(msg.RawRepresentation); - } - - [Fact] - public async Task InjectClientMessage_ResponseCreate_SerializedCorrectly() - { - var message = new RealtimeClientResponseCreateMessage - { - EventId = "client_evt_001", - Instructions = "Be concise", - MaxOutputTokens = 1024, - OutputModalities = new List { "text", "audio" }, - ExcludeFromConversation = true, - OutputVoice = "alloy", - OutputAudioOptions = new RealtimeAudioFormat("audio/pcm", 24000), - Metadata = new AdditionalPropertiesDictionary { ["key1"] = "value1" }, - ToolMode = ChatToolMode.Auto, - }; - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - Assert.Equal("response.create", sent["type"]?.GetValue()); - Assert.Equal("client_evt_001", sent["event_id"]?.GetValue()); - var response = sent["response"]!.AsObject(); - Assert.Equal("Be concise", response["instructions"]?.GetValue()); - Assert.Equal(1024, response["max_output_tokens"]?.GetValue()); - Assert.Equal("none", response["conversation"]?.GetValue()); - Assert.Equal("auto", response["tool_choice"]?.GetValue()); - var modalities = response["output_modalities"]!.AsArray(); - Assert.Equal(2, modalities.Count); - Assert.Equal("text", modalities[0]?.GetValue()); - Assert.Equal("audio", modalities[1]?.GetValue()); - Assert.Equal("value1", response["metadata"]!.AsObject()["key1"]?.GetValue()); - } - - [Fact] - public async Task InjectClientMessage_ConversationItemCreate_Message_SerializedCorrectly() - { - var contents = new List { new TextContent("Hello") }; - var item = new RealtimeContentItem(contents, "item_001", ChatRole.User); - var message = new RealtimeClientConversationItemCreateMessage(item, "prev_item_001"); - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - Assert.Equal("conversation.item.create", sent["type"]?.GetValue()); - Assert.Equal("prev_item_001", sent["previous_item_id"]?.GetValue()); - var itemObj = sent["item"]!.AsObject(); - Assert.Equal("message", itemObj["type"]?.GetValue()); - Assert.Equal("item_001", itemObj["id"]?.GetValue()); - Assert.Equal("user", itemObj["role"]?.GetValue()); - var contentArray = itemObj["content"]!.AsArray(); - Assert.Single(contentArray); - Assert.Equal("input_text", contentArray[0]!["type"]?.GetValue()); - Assert.Equal("Hello", contentArray[0]!["text"]?.GetValue()); - } - - [Fact] - public async Task InjectClientMessage_ConversationItemCreate_FunctionResult_SerializedCorrectly() - { - var functionResult = new FunctionResultContent("call_001", "Sunny, 72F"); - var item = new RealtimeContentItem(new List { functionResult }, "item_002"); - var message = new RealtimeClientConversationItemCreateMessage(item); - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - Assert.Equal("conversation.item.create", sent["type"]?.GetValue()); - var itemObj = sent["item"]!.AsObject(); - Assert.Equal("function_call_output", itemObj["type"]?.GetValue()); - Assert.Equal("call_001", itemObj["call_id"]?.GetValue()); - } - - [Fact] - public async Task InjectClientMessage_ConversationItemCreate_FunctionCall_SerializedCorrectly() - { - var functionCall = new FunctionCallContent("call_002", "get_weather", new Dictionary { ["city"] = "Seattle" }); - var item = new RealtimeContentItem(new List { functionCall }, "item_003"); - var message = new RealtimeClientConversationItemCreateMessage(item); - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - var itemObj = sent["item"]!.AsObject(); - Assert.Equal("function_call", itemObj["type"]?.GetValue()); - Assert.Equal("call_002", itemObj["call_id"]?.GetValue()); - Assert.Equal("get_weather", itemObj["name"]?.GetValue()); - Assert.NotNull(itemObj["arguments"]); - } - - [Fact] - public async Task InjectClientMessage_AudioBufferCommit_SerializedCorrectly() - { - await _session.InjectClientMessageAsync(new RealtimeClientInputAudioBufferCommitMessage()); - var sent = await ReadSentMessageAsync(); - Assert.Equal("input_audio_buffer.commit", sent["type"]?.GetValue()); - } - - [Fact] - public async Task InjectClientMessage_AudioBufferAppend_SerializedCorrectly() - { - var audioBytes = new byte[] { 0x01, 0x02, 0x03, 0x04 }; - var dataContent = new DataContent(audioBytes, "audio/pcm"); - var message = new RealtimeClientInputAudioBufferAppendMessage(dataContent); - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - Assert.Equal("input_audio_buffer.append", sent["type"]?.GetValue()); - var audioBase64 = sent["audio"]!.GetValue(); - Assert.NotNull(audioBase64); - var decoded = Convert.FromBase64String(audioBase64); - Assert.Equal(audioBytes, decoded); - } - - [Fact] - public async Task InjectClientMessage_RawRepresentation_String_SerializedCorrectly() - { - var rawJson = """{"type":"custom.event","data":"test"}"""; - var message = new RealtimeClientMessage { RawRepresentation = rawJson }; - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - Assert.Equal("custom.event", sent["type"]?.GetValue()); - Assert.Equal("test", sent["data"]?.GetValue()); - } - - [Fact] - public async Task InjectClientMessage_RawRepresentation_String_WithEventId_PreservesEventId() - { - var rawJson = """{"type":"custom.event","data":"test"}"""; - var message = new RealtimeClientMessage { RawRepresentation = rawJson, EventId = "evt_custom_001" }; - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - Assert.Equal("custom.event", sent["type"]?.GetValue()); - Assert.Equal("evt_custom_001", sent["event_id"]?.GetValue()); - } - - [Fact] - public async Task InjectClientMessage_RawRepresentation_String_WithEventIdInJson_DoesNotOverwrite() - { - var rawJson = """{"type":"custom.event","event_id":"evt_from_json"}"""; - var message = new RealtimeClientMessage { RawRepresentation = rawJson, EventId = "evt_from_property" }; - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - // The event_id already present in the raw JSON should take precedence. - Assert.Equal("evt_from_json", sent["event_id"]?.GetValue()); - } - - [Fact] - public async Task InjectClientMessage_RawRepresentation_JsonObject_SerializedCorrectly() - { - var rawObj = new JsonObject { ["type"] = "custom.event2", ["payload"] = "data" }; - var message = new RealtimeClientMessage { RawRepresentation = rawObj }; - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - Assert.Equal("custom.event2", sent["type"]?.GetValue()); - Assert.Equal("data", sent["payload"]?.GetValue()); - } - - #region MCP Tool Serialization Tests - - [Fact] - public async Task SessionUpdate_HostedMcpServerTool_WithUrl_SerializedCorrectly() - { - var mcpTool = new HostedMcpServerTool("my-server", new Uri("https://mcp.example.com/api")) - { - ServerDescription = "A test MCP server", - AllowedTools = new List { "search", "lookup" }, - ApprovalMode = HostedMcpServerToolApprovalMode.NeverRequire, - }; - - var options = new RealtimeSessionOptions - { - SessionKind = RealtimeSessionKind.Realtime, - Tools = [mcpTool], - }; - - await _session.UpdateAsync(options); - var sent = await ReadSentMessageAsync(); - - var sessionObj = sent["session"]!.AsObject(); - var toolsArray = sessionObj["tools"]!.AsArray(); - Assert.Single(toolsArray); - - var toolObj = toolsArray[0]!.AsObject(); - Assert.Equal("mcp", toolObj["type"]?.GetValue()); - Assert.Equal("my-server", toolObj["server_label"]?.GetValue()); - Assert.Equal("https://mcp.example.com/api", toolObj["server_url"]?.GetValue()); - Assert.Equal("A test MCP server", toolObj["server_description"]?.GetValue()); - Assert.Equal("never", toolObj["require_approval"]?.GetValue()); - Assert.Null(toolObj["connector_id"]); - - var allowedTools = toolObj["allowed_tools"]!.AsArray(); - Assert.Equal(2, allowedTools.Count); - Assert.Equal("search", allowedTools[0]?.GetValue()); - Assert.Equal("lookup", allowedTools[1]?.GetValue()); - } - - [Fact] - public async Task SessionUpdate_HostedMcpServerTool_WithConnectorId_SerializedCorrectly() - { - var mcpTool = new HostedMcpServerTool("connector-server", "my-connector-id") - { - AuthorizationToken = "test-token-123", - ApprovalMode = HostedMcpServerToolApprovalMode.AlwaysRequire, - }; - - var options = new RealtimeSessionOptions - { - SessionKind = RealtimeSessionKind.Realtime, - Tools = [mcpTool], - }; - - await _session.UpdateAsync(options); - var sent = await ReadSentMessageAsync(); - - var sessionObj = sent["session"]!.AsObject(); - var toolObj = sessionObj["tools"]!.AsArray()[0]!.AsObject(); - Assert.Equal("mcp", toolObj["type"]?.GetValue()); - Assert.Equal("connector-server", toolObj["server_label"]?.GetValue()); - Assert.Equal("my-connector-id", toolObj["connector_id"]?.GetValue()); - Assert.Null(toolObj["server_url"]); - Assert.Equal("always", toolObj["require_approval"]?.GetValue()); - - var authObj = toolObj["authorization"]!.AsObject(); - Assert.Equal("test-token-123", authObj["token"]?.GetValue()); - } - - [Fact] - public async Task SessionUpdate_HostedMcpServerTool_WithHeaders_SerializedCorrectly() - { - var mcpTool = new HostedMcpServerTool("header-server", new Uri("https://mcp.example.com/")); - mcpTool.Headers["X-Custom"] = "custom-value"; - mcpTool.Headers["Authorization"] = "Bearer my-token"; - - var options = new RealtimeSessionOptions - { - SessionKind = RealtimeSessionKind.Realtime, - Tools = [mcpTool], - }; - - await _session.UpdateAsync(options); - var sent = await ReadSentMessageAsync(); - - var toolObj = sent["session"]!.AsObject()["tools"]!.AsArray()[0]!.AsObject(); - var headersObj = toolObj["headers"]!.AsObject(); - Assert.Equal("custom-value", headersObj["X-Custom"]?.GetValue()); - Assert.Equal("Bearer my-token", headersObj["Authorization"]?.GetValue()); - } - - [Fact] - public async Task SessionUpdate_HostedMcpServerTool_SpecificApproval_SerializedCorrectly() - { - var mcpTool = new HostedMcpServerTool("specific-server", new Uri("https://mcp.example.com/")) - { - ApprovalMode = HostedMcpServerToolApprovalMode.RequireSpecific( - alwaysRequireApprovalToolNames: new List { "delete_file" }, - neverRequireApprovalToolNames: new List { "read_file", "list_files" }), - }; - - var options = new RealtimeSessionOptions - { - SessionKind = RealtimeSessionKind.Realtime, - Tools = [mcpTool], - }; - - await _session.UpdateAsync(options); - var sent = await ReadSentMessageAsync(); - - var toolObj = sent["session"]!.AsObject()["tools"]!.AsArray()[0]!.AsObject(); - var approvalObj = toolObj["require_approval"]!.AsObject(); - - var alwaysObj = approvalObj["always"]!.AsObject(); - var alwaysNames = alwaysObj["tool_names"]!.AsArray(); - Assert.Single(alwaysNames); - Assert.Equal("delete_file", alwaysNames[0]?.GetValue()); - - var neverObj = approvalObj["never"]!.AsObject(); - var neverNames = neverObj["tool_names"]!.AsArray(); - Assert.Equal(2, neverNames.Count); - Assert.Equal("read_file", neverNames[0]?.GetValue()); - Assert.Equal("list_files", neverNames[1]?.GetValue()); - } - - [Fact] - public async Task SessionUpdate_MixedTools_AIFunctionAndMcpTool_SerializedCorrectly() - { - var aiFunction = AIFunctionFactory.Create(() => "result", "test_func", "A test function"); - var mcpTool = new HostedMcpServerTool("mcp-server", new Uri("https://mcp.example.com/")); - - var options = new RealtimeSessionOptions - { - SessionKind = RealtimeSessionKind.Realtime, - Tools = [aiFunction, mcpTool], - }; - - await _session.UpdateAsync(options); - var sent = await ReadSentMessageAsync(); - - var toolsArray = sent["session"]!.AsObject()["tools"]!.AsArray(); - Assert.Equal(2, toolsArray.Count); - Assert.Equal("function", toolsArray[0]!["type"]?.GetValue()); - Assert.Equal("mcp", toolsArray[1]!["type"]?.GetValue()); - } - - [Fact] - public async Task ResponseCreate_HostedMcpServerTool_InToolsList_SerializedCorrectly() - { - var mcpTool = new HostedMcpServerTool("resp-server", new Uri("https://mcp.example.com/")) - { - ServerDescription = "Response-level MCP", - ApprovalMode = HostedMcpServerToolApprovalMode.NeverRequire, - }; - - var message = new RealtimeClientResponseCreateMessage - { - Tools = [mcpTool], - }; - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - var responseObj = sent["response"]!.AsObject(); - var toolsArray = responseObj["tools"]!.AsArray(); - Assert.Single(toolsArray); - var toolObj = toolsArray[0]!.AsObject(); - Assert.Equal("mcp", toolObj["type"]?.GetValue()); - Assert.Equal("resp-server", toolObj["server_label"]?.GetValue()); - Assert.Equal("https://mcp.example.com/", toolObj["server_url"]?.GetValue()); - Assert.Equal("Response-level MCP", toolObj["server_description"]?.GetValue()); - Assert.Equal("never", toolObj["require_approval"]?.GetValue()); - } - - #endregion - - #region MCP Server Event Parsing Tests - - [Fact] - public async Task ProcessServerEvent_McpCallCompleted_ParsedCorrectly() - { - var json = """{"type":"mcp_call.completed","event_id":"evt_mcp_001","response_id":"resp_010","output_index":0,"item":{"type":"mcp_call","id":"mcp_call_001","name":"search","server_label":"my-mcp","arguments":"{\"query\":\"weather\"}","output":"Sunny, 72F"}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.McpCallCompleted, outputMsg.Type); - Assert.Equal("evt_mcp_001", outputMsg.EventId); - Assert.Equal("resp_010", outputMsg.ResponseId); - Assert.Equal(0, outputMsg.OutputIndex); - - Assert.NotNull(outputMsg.Item); - Assert.Equal("mcp_call_001", outputMsg.Item.Id); - Assert.Equal(2, outputMsg.Item.Contents.Count); - - var callContent = Assert.IsType(outputMsg.Item.Contents[0]); - Assert.Equal("mcp_call_001", callContent.CallId); - Assert.Equal("search", callContent.ToolName); - Assert.Equal("my-mcp", callContent.ServerName); - Assert.NotNull(callContent.Arguments); - Assert.Equal("weather", callContent.Arguments["query"]?.ToString()); - - var resultContent = Assert.IsType(outputMsg.Item.Contents[1]); - Assert.Equal("mcp_call_001", resultContent.CallId); - Assert.NotNull(resultContent.Output); - var textOutput = Assert.IsType(resultContent.Output[0]); - Assert.Equal("Sunny, 72F", textOutput.Text); - } - - [Fact] - public async Task ProcessServerEvent_McpCallFailed_ParsedWithError() - { - var json = """{"type":"mcp_call.failed","event_id":"evt_mcp_002","item":{"type":"mcp_call","id":"mcp_call_002","name":"delete_file","server_label":"file-server","arguments":"{\"path\":\"/tmp/x\"}","error":{"type":"tool_execution_error","message":"Permission denied"}}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.McpCallFailed, outputMsg.Type); - - Assert.NotNull(outputMsg.Item); - Assert.Equal(2, outputMsg.Item.Contents.Count); - - var callContent = Assert.IsType(outputMsg.Item.Contents[0]); - Assert.Equal("delete_file", callContent.ToolName); - Assert.Equal("file-server", callContent.ServerName); - - var resultContent = Assert.IsType(outputMsg.Item.Contents[1]); - var errorContent = Assert.IsType(resultContent.Output![0]); - Assert.Contains("Permission denied", errorContent.Message); - } - - [Fact] - public async Task ProcessServerEvent_McpCallInProgress_ParsedCorrectly() - { - var json = """{"type":"mcp_call.in_progress","event_id":"evt_mcp_003","item_id":"mcp_call_003"}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.McpCallInProgress, outputMsg.Type); - Assert.Equal("evt_mcp_003", outputMsg.EventId); - Assert.NotNull(outputMsg.Item); - Assert.Equal("mcp_call_003", outputMsg.Item.Id); - } - - [Fact] - public async Task ProcessServerEvent_McpListToolsCompleted_ParsedWithToolsList() - { - var json = """{"type":"mcp_list_tools.completed","event_id":"evt_mcp_004","item_id":"list_001","item":{"type":"mcp_list_tools","id":"list_001","server_label":"my-mcp","tools":[{"name":"search","description":"Search the web","input_schema":{"type":"object"}},{"name":"lookup","description":"Lookup a value"}]}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.McpListToolsCompleted, outputMsg.Type); - Assert.Equal("evt_mcp_004", outputMsg.EventId); - Assert.NotNull(outputMsg.RawRepresentation); - - Assert.NotNull(outputMsg.Item); - Assert.Equal("list_001", outputMsg.Item.Id); - Assert.Equal(2, outputMsg.Item.Contents.Count); - - var tool1 = Assert.IsType(outputMsg.Item.Contents[0]); - Assert.Equal("search", tool1.ToolName); - Assert.Equal("my-mcp", tool1.ServerName); - - var tool2 = Assert.IsType(outputMsg.Item.Contents[1]); - Assert.Equal("lookup", tool2.ToolName); - Assert.Equal("my-mcp", tool2.ServerName); - } - - [Fact] - public async Task ProcessServerEvent_McpListToolsInProgress_ParsedWithItemId() - { - var json = """{"type":"mcp_list_tools.in_progress","event_id":"evt_mcp_005","item_id":"list_002"}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.McpListToolsInProgress, outputMsg.Type); - Assert.NotNull(outputMsg.Item); - Assert.Equal("list_002", outputMsg.Item.Id); - } - - [Fact] - public async Task ProcessServerEvent_McpListToolsFailed_ParsedWithItemId() - { - var json = """{"type":"mcp_list_tools.failed","event_id":"evt_mcp_006","item_id":"list_003"}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.McpListToolsFailed, outputMsg.Type); - Assert.NotNull(outputMsg.Item); - Assert.Equal("list_003", outputMsg.Item.Id); - } - - [Fact] - public async Task ProcessServerEvent_McpApprovalRequest_ParsedCorrectly() - { - var json = """{"type":"conversation.item.added","event_id":"evt_mcp_007","item":{"type":"mcp_approval_request","id":"approval_001","name":"charge_card","server_label":"payment-mcp","arguments":"{\"amount\":99.99}"}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.NotNull(outputMsg.Item); - Assert.Single(outputMsg.Item.Contents); - - var approvalRequest = Assert.IsType(outputMsg.Item.Contents[0]); - Assert.Equal("approval_001", approvalRequest.Id); - Assert.Equal("charge_card", approvalRequest.ToolCall.ToolName); - Assert.Equal("payment-mcp", approvalRequest.ToolCall.ServerName); - Assert.NotNull(approvalRequest.ToolCall.Arguments); - } - - [Fact] - public async Task ProcessServerEvent_ConversationItemDone_WithMcpCall_ParsedCorrectly() - { - var json = """{"type":"conversation.item.done","event_id":"evt_mcp_008","item":{"type":"mcp_call","id":"mcp_done_001","name":"read_file","server_label":"fs-server","arguments":"{\"path\":\"/readme.md\"}","output":"# Hello"}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.ResponseOutputItemDone, outputMsg.Type); - Assert.NotNull(outputMsg.Item); - - var callContent = Assert.IsType(outputMsg.Item.Contents[0]); - Assert.Equal("read_file", callContent.ToolName); - - var resultContent = Assert.IsType(outputMsg.Item.Contents[1]); - var textOutput = Assert.IsType(resultContent.Output![0]); - Assert.Equal("# Hello", textOutput.Text); - } - - [Fact] - public async Task ProcessServerEvent_ConversationItemAdded_WithRegularMessage_ParsedCorrectly() - { - var json = """{"type":"conversation.item.added","event_id":"evt_conv_001","item":{"type":"message","id":"msg_conv_001","role":"user","content":[{"type":"input_text","text":"Hello"}]}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.ResponseOutputItemAdded, outputMsg.Type); - Assert.NotNull(outputMsg.Item); - Assert.Equal("msg_conv_001", outputMsg.Item.Id); - Assert.Equal(ChatRole.User, outputMsg.Item.Role); - var textContent = Assert.IsType(outputMsg.Item.Contents[0]); - Assert.Equal("Hello", textContent.Text); - } - - [Fact] - public async Task ProcessServerEvent_ConversationItemAdded_WithUnknownType_ReturnsRawContent() - { - var json = """{"type":"conversation.item.added","event_id":"evt_conv_002","item":{"type":"unknown_item_type","id":"unknown_001"}}"""; - - await SendServerMessageAsync(json); - var msg = await ReadNextServerMessageAsync(); - - var outputMsg = Assert.IsType(msg); - Assert.Equal(RealtimeServerMessageType.RawContentOnly, outputMsg.Type); - Assert.NotNull(outputMsg.RawRepresentation); - } - - #endregion - - #region MCP Approval Response Sending Tests - - [Fact] - public async Task InjectClientMessage_McpApprovalResponse_Approved_SerializedCorrectly() - { - var approvalResponse = new McpServerToolApprovalResponseContent("approval_001", approved: true); - var item = new RealtimeContentItem(new List { approvalResponse }, "resp_item_001"); - var message = new RealtimeClientConversationItemCreateMessage(item); - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - Assert.Equal("conversation.item.create", sent["type"]?.GetValue()); - var itemObj = sent["item"]!.AsObject(); - Assert.Equal("mcp_approval_response", itemObj["type"]?.GetValue()); - Assert.Equal("approval_001", itemObj["approval_request_id"]?.GetValue()); - Assert.True(itemObj["approve"]?.GetValue()); - } - - [Fact] - public async Task InjectClientMessage_McpApprovalResponse_Rejected_SerializedCorrectly() - { - var approvalResponse = new McpServerToolApprovalResponseContent("approval_002", approved: false); - var item = new RealtimeContentItem(new List { approvalResponse }, "resp_item_002"); - var message = new RealtimeClientConversationItemCreateMessage(item); - - await _session.InjectClientMessageAsync(message); - var sent = await ReadSentMessageAsync(); - - var itemObj = sent["item"]!.AsObject(); - Assert.Equal("mcp_approval_response", itemObj["type"]?.GetValue()); - Assert.Equal("approval_002", itemObj["approval_request_id"]?.GetValue()); - Assert.False(itemObj["approve"]?.GetValue()); - } - - #endregion - - private async Task SendServerMessageAsync(string json) - { - var bytes = Encoding.UTF8.GetBytes(json); - await _serverWebSocket.SendAsync(new ArraySegment(bytes), WebSocketMessageType.Text, true, CancellationToken.None); - } - - private async Task ReadSentMessageAsync() - { - var buffer = new byte[1024 * 16]; - var result = await _serverWebSocket.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); - var json = Encoding.UTF8.GetString(buffer, 0, result.Count); - return JsonSerializer.Deserialize(json)!; - } - - private async Task ReadNextServerMessageAsync() - { - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - var enumerator = _session.GetStreamingResponseAsync(EmptyUpdatesAsync(), cts.Token) - .GetAsyncEnumerator(cts.Token); - try - { - if (await enumerator.MoveNextAsync().ConfigureAwait(false)) - { - return enumerator.Current; - } - } - finally - { - await enumerator.DisposeAsync().ConfigureAwait(false); - } - - throw new InvalidOperationException("No server message received within timeout"); - } - - private static async IAsyncEnumerable EmptyUpdatesAsync( - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false); - yield break; - } - - /// - /// A WebSocket implementation backed by channels, used for in-process testing without a real network connection. - /// - internal sealed class ChannelWebSocket : WebSocket - { - private readonly ChannelReader<(byte[] data, WebSocketMessageType messageType, bool endOfMessage)> _reader; - private readonly ChannelWriter<(byte[] data, WebSocketMessageType messageType, bool endOfMessage)> _writer; - private WebSocketState _state = WebSocketState.Open; - - public ChannelWebSocket( - ChannelReader<(byte[] data, WebSocketMessageType messageType, bool endOfMessage)> reader, - ChannelWriter<(byte[] data, WebSocketMessageType messageType, bool endOfMessage)> writer) - { - _reader = reader; - _writer = writer; - } - - public override WebSocketCloseStatus? CloseStatus => _state == WebSocketState.Closed ? WebSocketCloseStatus.NormalClosure : null; - public override string? CloseStatusDescription => null; - public override WebSocketState State => _state; - public override string? SubProtocol => null; - - public override async Task ReceiveAsync(ArraySegment buffer, CancellationToken cancellationToken) - { - var (data, type, endOfMessage) = await _reader.ReadAsync(cancellationToken).ConfigureAwait(false); - - if (type == WebSocketMessageType.Close) - { - _state = WebSocketState.CloseReceived; - return new WebSocketReceiveResult(0, WebSocketMessageType.Close, true, WebSocketCloseStatus.NormalClosure, null); - } - - var count = Math.Min(data.Length, buffer.Count); - Array.Copy(data, 0, buffer.Array!, buffer.Offset, count); - return new WebSocketReceiveResult(count, type, endOfMessage); - } - - public override async Task SendAsync(ArraySegment buffer, WebSocketMessageType messageType, bool endOfMessage, CancellationToken cancellationToken) - { - var data = new byte[buffer.Count]; - Array.Copy(buffer.Array!, buffer.Offset, data, 0, buffer.Count); - await _writer.WriteAsync((data, messageType, endOfMessage), cancellationToken).ConfigureAwait(false); - } - - public override Task CloseAsync(WebSocketCloseStatus closeStatus, string? statusDescription, CancellationToken cancellationToken) - { - _state = WebSocketState.Closed; - _writer.TryComplete(); - return Task.CompletedTask; - } - - public override Task CloseOutputAsync(WebSocketCloseStatus closeStatus, string? statusDescription, CancellationToken cancellationToken) - { - _state = WebSocketState.CloseSent; - _writer.TryComplete(); - return Task.CompletedTask; - } - - public override void Abort() - { - _state = WebSocketState.Aborted; - _writer.TryComplete(); - } - - public override void Dispose() - { - _state = WebSocketState.Closed; - _writer.TryComplete(); - } - } -} From 5845991cf512b76f167323e85c8f51c878320683 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 09:51:47 -0800 Subject: [PATCH 19/92] Add RawRepresentationFactory to RealtimeSessionOptions Add Func? RawRepresentationFactory property following the same pattern used by ChatOptions, EmbeddingGenerationOptions, and other abstraction options types. Add note in OpenAIRealtimeSession to consume the factory when switching to the OpenAI SDK. --- .../Realtime/RealtimeSessionOptions.cs | 22 +++++++++++++++++++ .../OpenAIRealtimeSession.cs | 4 ++++ 2 files changed, 26 insertions(+) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs index c8c0d3fb236..7e52db231c1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs @@ -1,8 +1,10 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; using Microsoft.Shared.DiagnosticIds; namespace Microsoft.Extensions.AI; @@ -136,4 +138,24 @@ public class RealtimeSessionOptions /// This property is only used if is not set to true. /// public object? TracingMetadata { get; set; } + + /// + /// Gets or sets a callback responsible for creating the raw representation of the session options from an underlying implementation. + /// + /// + /// The underlying implementation might have its own representation of options. + /// When is invoked with a , + /// that implementation might convert the provided options into its own representation in order to use it while + /// performing the operation. For situations where a consumer knows which concrete + /// is being used and how it represents options, a new instance of that implementation-specific options type can be + /// returned by this callback for the implementation to use, instead of creating a + /// new instance. Such implementations might mutate the supplied options instance further based on other settings + /// supplied on this instance or from other inputs. + /// Therefore, it is strongly recommended to not return shared instances and instead make the callback return + /// a new instance on each call. + /// This is typically used to set an implementation-specific setting that isn't otherwise exposed from the strongly typed + /// properties on . + /// + [JsonIgnore] + public Func? RawRepresentationFactory { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index cc078142c6a..4c0a34b859e 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -118,6 +118,10 @@ public async Task UpdateAsync(RealtimeSessionOptions options, CancellationToken { _ = Throw.IfNull(options); + // Note: When switching to the OpenAI SDK for serialization, consume options.RawRepresentationFactory + // here to allow callers to provide a pre-configured SDK-specific options instance, following the + // same pattern used by OpenAIChatClient and other provider implementations. + var sessionElement = new JsonObject { ["type"] = "session.update", From 63531b5e16015e1313fe72de6fd49451ba71fbfb Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 10:09:26 -0800 Subject: [PATCH 20/92] Remove OpenAI-specific tracing properties from RealtimeSessionOptions Remove EnableAutoTracing, TracingGroupId, TracingWorkflowName, and TracingMetadata from the abstraction layer. These are OpenAI-specific and should be configured via RawRepresentationFactory when the OpenAI SDK dependency is added. --- .../Realtime/RealtimeSessionOptions.cs | 30 ------------------- .../OpenAIRealtimeSession.cs | 4 --- .../Realtime/RealtimeSessionOptionsTests.cs | 13 -------- 3 files changed, 47 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs index 7e52db231c1..0812757c178 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs @@ -109,36 +109,6 @@ public class RealtimeSessionOptions /// public IList? Tools { get; set; } - /// - /// Gets or sets a value indicating whether to enable automatic tracing for the session. - /// if enabled, will create a trace for the session with default values for the workflow name, group id, and metadata. - /// - public bool EnableAutoTracing { get; set; } - - /// - /// Gets or sets the group ID for tracing. - /// - /// - /// This property is only used if is not set to true. - /// - public string? TracingGroupId { get; set; } - - /// - /// Gets or sets the workflow name for tracing. - /// - /// - /// This property is only used if is not set to true. - /// - public string? TracingWorkflowName { get; set; } - - /// - /// Gets or sets arbitrary metadata to attach to this trace to enable filtering. - /// - /// - /// This property is only used if is not set to true. - /// - public object? TracingMetadata { get; set; } - /// /// Gets or sets a callback responsible for creating the raw representation of the session options from an underlying implementation. /// diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 4c0a34b859e..118f01324e5 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -1366,10 +1366,6 @@ [new McpServerToolApprovalRequestContent(approvalId, toolCall) { RawRepresentati newOptions.AIFunction = Options.AIFunction; newOptions.HostedMcpServerTool = Options.HostedMcpServerTool; newOptions.ToolMode = Options.ToolMode; - newOptions.EnableAutoTracing = Options.EnableAutoTracing; - newOptions.TracingGroupId = Options.TracingGroupId; - newOptions.TracingWorkflowName = Options.TracingWorkflowName; - newOptions.TracingMetadata = Options.TracingMetadata; } Options = newOptions; diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs index 57c588801b9..f789b4c60dd 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs @@ -31,10 +31,6 @@ public void Constructor_Parameterless_PropsDefaulted() Assert.Null(options.AIFunction); Assert.Null(options.HostedMcpServerTool); Assert.Null(options.Tools); - Assert.False(options.EnableAutoTracing); - Assert.Null(options.TracingGroupId); - Assert.Null(options.TracingWorkflowName); - Assert.Null(options.TracingMetadata); } [Fact] @@ -46,7 +42,6 @@ public void Properties_Roundtrip() var outputFormat = new RealtimeAudioFormat("audio/pcm", 24000); List modalities = ["text", "audio"]; List tools = [AIFunctionFactory.Create(() => 42)]; - var tracingMetadata = new { key = "value" }; var transcriptionOptions = new TranscriptionOptions { SpeechLanguage = "en", ModelId = "whisper-1", Prompt = "greeting" }; var vad = new VoiceActivityDetection { CreateResponse = true, InterruptResponse = true }; @@ -64,10 +59,6 @@ public void Properties_Roundtrip() options.OutputModalities = modalities; options.ToolMode = ChatToolMode.Auto; options.Tools = tools; - options.EnableAutoTracing = true; - options.TracingGroupId = "group-1"; - options.TracingWorkflowName = "workflow-1"; - options.TracingMetadata = tracingMetadata; Assert.Equal(RealtimeSessionKind.Transcription, options.SessionKind); Assert.Equal("gpt-4-realtime", options.Model); @@ -83,10 +74,6 @@ public void Properties_Roundtrip() Assert.Same(modalities, options.OutputModalities); Assert.Equal(ChatToolMode.Auto, options.ToolMode); Assert.Same(tools, options.Tools); - Assert.True(options.EnableAutoTracing); - Assert.Equal("group-1", options.TracingGroupId); - Assert.Equal("workflow-1", options.TracingWorkflowName); - Assert.Same(tracingMetadata, options.TracingMetadata); } [Fact] From f333eaa7e47193ce68edb32e09ab2c3cf047a4fb Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 11:03:12 -0800 Subject: [PATCH 21/92] Remove AIFunction/HostedMcpServerTool properties, use ChatToolMode Remove redundant AIFunction and HostedMcpServerTool properties from RealtimeSessionOptions and RealtimeClientResponseCreateMessage. Callers should use ChatToolMode.RequireSpecific(functionName) instead. Update OpenAI serialization to emit structured tool_choice JSON object when RequireSpecific is used. Update OpenTelemetry and tests accordingly. --- .../RealtimeClientResponseCreateMessage.cs | 13 --------- .../Realtime/RealtimeSessionOptions.cs | 19 ------------- .../OpenAIRealtimeSession.cs | 27 +++++-------------- .../Realtime/OpenTelemetryRealtimeSession.cs | 13 +-------- .../Realtime/RealtimeClientMessageTests.cs | 2 -- .../Realtime/RealtimeSessionOptionsTests.cs | 2 -- .../OpenTelemetryRealtimeSessionTests.cs | 14 +++------- 7 files changed, 11 insertions(+), 79 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs index 07dd8988bce..a840195b4fd 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs @@ -65,21 +65,8 @@ public RealtimeClientResponseCreateMessage() /// /// Gets or sets the tool choice mode for the response. /// - /// - /// If AIFunction or HostedMcpServerTool is specified, this value will be ignored. - /// public ChatToolMode? ToolMode { get; set; } - /// - /// Gets or sets the AI function to use for the response. - /// - public AIFunction? AIFunction { get; set; } - - /// - /// Gets or sets the hosted MCP server tool configuration for the response. - /// - public HostedMcpServerTool? HostedMcpServerTool { get; set; } - /// /// Gets or sets the AI tools available for generating the response. /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs index 0812757c178..6180a12ab87 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs @@ -83,27 +83,8 @@ public class RealtimeSessionOptions /// /// Gets or sets the tool choice mode for the session. /// - /// - /// If or is specified, this value will be ignored. - /// public ChatToolMode? ToolMode { get; set; } - /// - /// Gets or sets the AI function to use for the response. - /// - /// - /// If specified, the will be ignored. - /// - public AIFunction? AIFunction { get; set; } - - /// - /// Gets or sets the name of the MCP tool to use for the response. - /// - /// - /// If specified, the will be ignored. - /// - public HostedMcpServerTool? HostedMcpServerTool { get; set; } - /// /// Gets or sets the AI tools available for generating the response. /// diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 118f01324e5..8266378e3fd 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -400,28 +400,15 @@ public async Task InjectClientMessageAsync(RealtimeClientMessage message, Cancel responseObj["output_modalities"] = CreateModalitiesArray(responseCreate.OutputModalities); } - if (responseCreate.AIFunction is not null) - { - responseObj["tool_choice"] = new JsonObject - { - ["type"] = "function", - ["name"] = responseCreate.AIFunction.Name, - }; - } - else if (responseCreate.HostedMcpServerTool is not null) - { - responseObj["tool_choice"] = new JsonObject - { - ["type"] = "mcp", - ["server_label"] = responseCreate.HostedMcpServerTool.ServerName, - ["name"] = responseCreate.HostedMcpServerTool.Name, - }; - } - else if (responseCreate.ToolMode is { } toolMode) + if (responseCreate.ToolMode is { } toolMode) { responseObj["tool_choice"] = toolMode switch { - RequiredChatToolMode r when r.RequiredFunctionName is not null => r.RequiredFunctionName, + RequiredChatToolMode r when r.RequiredFunctionName is not null => new JsonObject + { + ["type"] = "function", + ["name"] = r.RequiredFunctionName, + }, RequiredChatToolMode => "required", NoneChatToolMode => "none", _ => "auto", @@ -1363,8 +1350,6 @@ [new McpServerToolApprovalRequestContent(approvalId, toolCall) { RawRepresentati if (Options is not null) { newOptions.Tools = Options.Tools; - newOptions.AIFunction = Options.AIFunction; - newOptions.HostedMcpServerTool = Options.HostedMcpServerTool; newOptions.ToolMode = Options.ToolMode; } diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs index 65122718a2a..a559ce23973 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs @@ -808,19 +808,8 @@ private static string SerializeMessages(IEnumerable message } // Tool choice mode (custom attribute - not part of OTel GenAI spec) - // Priority: AIFunction > HostedMcpServerTool > ToolMode string? toolChoice = null; - if (options.AIFunction is { } aiFunc) - { - // When a specific AIFunction is forced, use its name - toolChoice = aiFunc.Name; - } - else if (options.HostedMcpServerTool is { } mcpTool) - { - // When a specific MCP tool is forced, use the server name (mcp:) - toolChoice = $"mcp:{mcpTool.ServerName}"; - } - else if (options.ToolMode is { } toolMode) + if (options.ToolMode is { } toolMode) { toolChoice = toolMode switch { diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs index 4a5847bd80a..a2d0530d39c 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs @@ -140,8 +140,6 @@ public void ResponseCreateMessage_DefaultProperties() Assert.Null(message.Metadata); Assert.Null(message.OutputModalities); Assert.Null(message.ToolMode); - Assert.Null(message.AIFunction); - Assert.Null(message.HostedMcpServerTool); Assert.Null(message.Tools); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs index f789b4c60dd..c7ed4204248 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs @@ -28,8 +28,6 @@ public void Constructor_Parameterless_PropsDefaulted() Assert.Null(options.MaxOutputTokens); Assert.Null(options.OutputModalities); Assert.Null(options.ToolMode); - Assert.Null(options.AIFunction); - Assert.Null(options.HostedMcpServerTool); Assert.Null(options.Tools); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs index 8b266605576..6111ee0fe8d 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs @@ -617,15 +617,12 @@ public async Task AIFunction_ForcedTool_Logged() .AddInMemoryExporter(activities) .Build(); - var forcedFunction = AIFunctionFactory.Create((string query) => query, "SpecificSearch", "Search with specific parameters"); - using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model", - AIFunction = forcedFunction, - ToolMode = ChatToolMode.Auto, // Should be ignored when AIFunction is set + ToolMode = ChatToolMode.RequireSpecific("SpecificSearch"), }, GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), }; @@ -647,7 +644,7 @@ public async Task AIFunction_ForcedTool_Logged() #pragma warning disable MEAI001 // Type is for evaluation purposes only [Fact] - public async Task HostedMcpServerTool_ForcedTool_Logged() + public async Task RequireAny_ToolMode_Logged() { var sourceName = Guid.NewGuid().ToString(); var activities = new List(); @@ -656,15 +653,12 @@ public async Task HostedMcpServerTool_ForcedTool_Logged() .AddInMemoryExporter(activities) .Build(); - var mcpTool = new HostedMcpServerTool("github-server", "https://mcp.github.com"); - using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model", - HostedMcpServerTool = mcpTool, - ToolMode = ChatToolMode.Auto, // Should be ignored when HostedMcpServerTool is set + ToolMode = ChatToolMode.RequireAny, }, GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), }; @@ -681,7 +675,7 @@ public async Task HostedMcpServerTool_ForcedTool_Logged() } var activity = Assert.Single(activities); - Assert.Equal("mcp:github-server", activity.GetTagItem("gen_ai.request.tool_choice")); + Assert.Equal("required", activity.GetTagItem("gen_ai.request.tool_choice")); } #pragma warning restore MEAI001 From 4a6947e865275c114de9636977044bffc215244e Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 11:19:56 -0800 Subject: [PATCH 22/92] Improve XML docs for Usage on response and transcription messages --- ...altimeServerInputAudioTranscriptionMessage.cs | 6 +++++- .../RealtimeServerResponseCreatedMessage.cs | 7 ++++++- temp_check.cs | 16 ++++++++++++++++ temp_check.csx | 12 ++++++++++++ 4 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 temp_check.cs create mode 100644 temp_check.csx diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs index 5ad03ab3997..ec011d3faee 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs @@ -43,8 +43,12 @@ public RealtimeServerInputAudioTranscriptionMessage(RealtimeServerMessageType ty public string? Transcription { get; set; } /// - /// Gets or sets the usage details for the transcription. + /// Gets or sets the transcription-specific usage, which is billed separately from the realtime model. /// + /// + /// This usage reflects the cost of the speech-to-text transcription and is billed according to the + /// ASR (Automatic Speech Recognition) model's pricing rather than the realtime model's pricing. + /// public UsageDetails? Usage { get; set; } /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs index ababa4a9947..1960f3b3d02 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs @@ -80,7 +80,12 @@ public RealtimeServerResponseCreatedMessage(RealtimeServerMessageType type) public ErrorContent? Error { get; set; } /// - /// Gets or sets the usage details for the response. + /// Gets or sets the per-response token usage for billing purposes. /// + /// + /// Populated when the response is complete (i.e., on ). + /// Input tokens include the entire conversation context, so they grow over successive turns + /// as previous output becomes input for later responses. + /// public UsageDetails? Usage { get; set; } } diff --git a/temp_check.cs b/temp_check.cs new file mode 100644 index 00000000000..6a7c6f60b04 --- /dev/null +++ b/temp_check.cs @@ -0,0 +1,16 @@ +using System; +using System.Reflection; +using System.Linq; + +class Program { + static void Main() { + var asm = Assembly.LoadFrom(@""Q:\.tools\.nuget\packages\openai\2.2.0-beta.4\lib\net6.0\OpenAI.dll""); + var realtimeTypes = asm.GetTypes() + .Where(t => t.IsPublic && t.Namespace != null && t.Namespace.Contains(""Realtime"")) + .OrderBy(t => t.FullName); + foreach (var t in realtimeTypes) + { + Console.WriteLine(t.FullName); + } + } +} diff --git a/temp_check.csx b/temp_check.csx new file mode 100644 index 00000000000..3dc6853a57f --- /dev/null +++ b/temp_check.csx @@ -0,0 +1,12 @@ +using System; +using System.Reflection; +using System.Linq; + +var asm = Assembly.LoadFrom(@"Q:\.tools\.nuget\packages\openai\2.2.0-beta.4\lib\net6.0\OpenAI.dll"); +var realtimeTypes = asm.GetTypes() + .Where(t => t.IsPublic && t.Namespace != null && t.Namespace.Contains("Realtime")) + .OrderBy(t => t.FullName); +foreach (var t in realtimeTypes) +{ + Console.WriteLine(t.FullName); +} From 29bbdca96e5a1ff7fd02fc76223d7f57bc198f9d Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 11:22:31 -0800 Subject: [PATCH 23/92] Remove leftover temp files --- temp_check.cs | 16 ---------------- temp_check.csx | 12 ------------ 2 files changed, 28 deletions(-) delete mode 100644 temp_check.cs delete mode 100644 temp_check.csx diff --git a/temp_check.cs b/temp_check.cs deleted file mode 100644 index 6a7c6f60b04..00000000000 --- a/temp_check.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System; -using System.Reflection; -using System.Linq; - -class Program { - static void Main() { - var asm = Assembly.LoadFrom(@""Q:\.tools\.nuget\packages\openai\2.2.0-beta.4\lib\net6.0\OpenAI.dll""); - var realtimeTypes = asm.GetTypes() - .Where(t => t.IsPublic && t.Namespace != null && t.Namespace.Contains(""Realtime"")) - .OrderBy(t => t.FullName); - foreach (var t in realtimeTypes) - { - Console.WriteLine(t.FullName); - } - } -} diff --git a/temp_check.csx b/temp_check.csx deleted file mode 100644 index 3dc6853a57f..00000000000 --- a/temp_check.csx +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Reflection; -using System.Linq; - -var asm = Assembly.LoadFrom(@"Q:\.tools\.nuget\packages\openai\2.2.0-beta.4\lib\net6.0\OpenAI.dll"); -var realtimeTypes = asm.GetTypes() - .Where(t => t.IsPublic && t.Namespace != null && t.Namespace.Contains("Realtime")) - .OrderBy(t => t.FullName); -foreach (var t in realtimeTypes) -{ - Console.WriteLine(t.FullName); -} From ab0e74d1d83db01114dfdc222360c86b5e3a5273 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 11:38:02 -0800 Subject: [PATCH 24/92] Improve ConversationId XML docs on RealtimeServerResponseCreatedMessage --- .../Realtime/RealtimeServerResponseCreatedMessage.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs index 1960f3b3d02..dcac8956737 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs @@ -40,6 +40,11 @@ public RealtimeServerResponseCreatedMessage(RealtimeServerMessageType type) /// /// Gets or sets the conversation ID associated with the response. /// + /// + /// Identifies which conversation within the session this response belongs to. + /// A session may have a default conversation to which items are automatically added, + /// or responses may be generated out-of-band (not associated with any conversation). + /// public string? ConversationId { get; set; } /// From 39a042d1a90fdca9a10bc36e10a6ee9180ca49a6 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 12:00:08 -0800 Subject: [PATCH 25/92] Split Text/Audio properties on RealtimeServerOutputTextAudioMessage Add separate Audio property for Base64-encoded audio data. Text is now only used for text and transcript content. Update OpenAI parser, OpenTelemetry session, and tests accordingly. --- .../RealtimeServerOutputTextAudioMessage.cs | 19 ++++++++++++++++--- .../OpenAIRealtimeSession.cs | 9 ++++++++- .../Realtime/OpenTelemetryRealtimeSession.cs | 2 +- .../Realtime/RealtimeServerMessageTests.cs | 1 + 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs index 7b1a5b3ea6d..be6b9137bfc 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs @@ -32,14 +32,27 @@ public RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType type) public int? ContentIndex { get; set; } /// - /// Gets or sets the text or audio delta, or the final text or audio once the output is complete. + /// Gets or sets the text delta or final text content. /// /// - /// if dealing with audio content, this property may contain Base64-encoded audio data. - /// With , usually will have null Text value. + /// Populated for , , + /// , and messages. + /// For audio messages ( and ), + /// use instead. /// public string? Text { get; set; } + /// + /// Gets or sets the Base64-encoded audio data delta or final audio content. + /// + /// + /// Populated for messages. + /// For , this is typically + /// as the final audio is not included; use the accumulated deltas instead. + /// For text content, use instead. + /// + public string? Audio { get; set; } + /// /// Gets or sets the ID of the item containing the content part whose text has been updated. /// diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 8266378e3fd..fa8ed439a5f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -1664,7 +1664,14 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (root.TryGetProperty("delta", out var deltaElement)) { - msg.Text = deltaElement.GetString(); + if (serverMessageType is RealtimeServerMessageType.OutputAudioDelta) + { + msg.Audio = deltaElement.GetString(); + } + else + { + msg.Text = deltaElement.GetString(); + } } if (msg.Text is null && root.TryGetProperty("transcript", out deltaElement)) diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs index a559ce23973..b49e90cc4b5 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs @@ -451,7 +451,7 @@ private static void AddOutputMessagesTag(Activity? activity, List ("text", textAudioMsg.Text), RealtimeServerMessageType.OutputAudioDelta or RealtimeServerMessageType.OutputAudioDone => - ("audio", string.IsNullOrEmpty(textAudioMsg.Text) ? "[audio data]" : textAudioMsg.Text), + ("audio", string.IsNullOrEmpty(textAudioMsg.Audio) ? "[audio data]" : textAudioMsg.Audio), RealtimeServerMessageType.OutputAudioTranscriptionDelta or RealtimeServerMessageType.OutputAudioTranscriptionDone => ("output_transcription", textAudioMsg.Text), _ => ("text", textAudioMsg.Text), diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs index 88eda1aac0c..06e1b155559 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs @@ -134,6 +134,7 @@ public void OutputTextAudioMessage_DefaultProperties() Assert.Null(message.ContentIndex); Assert.Null(message.Text); + Assert.Null(message.Audio); Assert.Null(message.ItemId); Assert.Null(message.OutputIndex); Assert.Null(message.ResponseId); From 46ae23dc4d79e085dbbcf88daa7be68d787208b9 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 12:29:45 -0800 Subject: [PATCH 26/92] Replace RealtimeServerMessageType enum with readonly struct Follow the ChatRole smart-enum pattern: readonly struct with string Value, IEquatable, operators, and JsonConverter. Providers can now define custom message types by constructing new instances. Update pattern-matching in OpenTelemetryRealtimeSession to use == comparisons instead of constant patterns. --- .../Realtime/RealtimeServerMessageType.cs | 239 +++++++++--------- .../OpenAIRealtimeSession.cs | 2 +- .../Realtime/OpenTelemetryRealtimeSession.cs | 57 +++-- 3 files changed, 158 insertions(+), 140 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs index 7f39ddb739e..7c722cf06a0 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs @@ -1,165 +1,160 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; +using System.ComponentModel; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; /// -/// Represents the type of a real-time response. -/// This is used to identify the response type being received from the model. +/// Represents the type of a real-time server message. +/// This is used to identify the message type being received from the model. /// +/// +/// Well-known message types are provided as static properties. Providers may define additional +/// message types by constructing new instances with custom values. +/// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public enum RealtimeServerMessageType +[JsonConverter(typeof(Converter))] +[DebuggerDisplay("{Value,nq}")] +public readonly struct RealtimeServerMessageType : IEquatable { - /// - /// Indicates that the response contains only raw content. - /// + /// Gets a message type indicating that the response contains only raw content. /// - /// This response type is to support extensibility for supporting custom content types not natively supported by the SDK. + /// This type supports extensibility for custom content types not natively supported by the SDK. /// - RawContentOnly, + public static RealtimeServerMessageType RawContentOnly { get; } = new("RawContentOnly"); - /// - /// Indicates the output of audio transcription for user audio written to the user audio buffer. - /// - /// - /// The type is used with this response type. - /// - InputAudioTranscriptionCompleted, + /// Gets a message type indicating the output of audio transcription for user audio written to the user audio buffer. + public static RealtimeServerMessageType InputAudioTranscriptionCompleted { get; } = new("InputAudioTranscriptionCompleted"); - /// - /// Indicates the text value of an input audio transcription content part is updated with incremental transcription results. - /// - /// - /// The type is used with this response type. - /// - InputAudioTranscriptionDelta, + /// Gets a message type indicating the text value of an input audio transcription content part is updated with incremental transcription results. + public static RealtimeServerMessageType InputAudioTranscriptionDelta { get; } = new("InputAudioTranscriptionDelta"); - /// - /// Indicates that the audio transcription for user audio written to the user audio buffer has failed. - /// - /// - /// The type is used with this response type. - /// - InputAudioTranscriptionFailed, + /// Gets a message type indicating that the audio transcription for user audio written to the user audio buffer has failed. + public static RealtimeServerMessageType InputAudioTranscriptionFailed { get; } = new("InputAudioTranscriptionFailed"); - /// - /// Indicates the output text update with incremental results response. - /// - /// - /// The type is used with this response type. - /// - OutputTextDelta, + /// Gets a message type indicating the output text update with incremental results. + public static RealtimeServerMessageType OutputTextDelta { get; } = new("OutputTextDelta"); - /// - /// Indicates the output text is complete. - /// - /// - /// The type is used with this response type. - /// - OutputTextDone, + /// Gets a message type indicating the output text is complete. + public static RealtimeServerMessageType OutputTextDone { get; } = new("OutputTextDone"); - /// - /// Indicates the model-generated transcription of audio output updated. - /// - /// - /// The type is used with this response type. - /// - OutputAudioTranscriptionDelta, + /// Gets a message type indicating the model-generated transcription of audio output updated. + public static RealtimeServerMessageType OutputAudioTranscriptionDelta { get; } = new("OutputAudioTranscriptionDelta"); - /// - /// Indicates the model-generated transcription of audio output is done streaming. - /// - /// - /// The type is used with this response type. - /// - OutputAudioTranscriptionDone, + /// Gets a message type indicating the model-generated transcription of audio output is done streaming. + public static RealtimeServerMessageType OutputAudioTranscriptionDone { get; } = new("OutputAudioTranscriptionDone"); - /// - /// Indicates the audio output updated. - /// - /// - /// The type is used with this response type. - /// - OutputAudioDelta, + /// Gets a message type indicating the audio output updated. + public static RealtimeServerMessageType OutputAudioDelta { get; } = new("OutputAudioDelta"); - /// - /// Indicates the audio output is done streaming. - /// - /// - /// The type is used with this response type. - /// - OutputAudioDone, + /// Gets a message type indicating the audio output is done streaming. + public static RealtimeServerMessageType OutputAudioDone { get; } = new("OutputAudioDone"); - /// - /// Indicates the response has completed. - /// - /// - /// The type is used with this response type. - /// - ResponseDone, + /// Gets a message type indicating the response has completed. + public static RealtimeServerMessageType ResponseDone { get; } = new("ResponseDone"); - /// - /// Indicates the response has been created. - /// - /// - /// The type is used with this response type. - /// - ResponseCreated, + /// Gets a message type indicating the response has been created. + public static RealtimeServerMessageType ResponseCreated { get; } = new("ResponseCreated"); - /// - /// Indicates an individual output item in the response has completed. - /// - /// - /// The type is used with this response type. - /// - ResponseOutputItemDone, + /// Gets a message type indicating an individual output item in the response has completed. + public static RealtimeServerMessageType ResponseOutputItemDone { get; } = new("ResponseOutputItemDone"); - /// - /// Indicates an individual output item has been added to the response. - /// - /// - /// The type is used with this response type. - /// - ResponseOutputItemAdded, + /// Gets a message type indicating an individual output item has been added to the response. + public static RealtimeServerMessageType ResponseOutputItemAdded { get; } = new("ResponseOutputItemAdded"); - /// - /// Indicates an error occurred while processing the request. - /// - /// - /// The type is used with this response type. - /// - Error, + /// Gets a message type indicating an error occurred while processing the request. + public static RealtimeServerMessageType Error { get; } = new("Error"); - /// - /// Indicates that an MCP tool call is in progress. - /// - McpCallInProgress, + /// Gets a message type indicating that an MCP tool call is in progress. + public static RealtimeServerMessageType McpCallInProgress { get; } = new("McpCallInProgress"); - /// - /// Indicates that an MCP tool call has completed. - /// - McpCallCompleted, + /// Gets a message type indicating that an MCP tool call has completed. + public static RealtimeServerMessageType McpCallCompleted { get; } = new("McpCallCompleted"); + + /// Gets a message type indicating that an MCP tool call has failed. + public static RealtimeServerMessageType McpCallFailed { get; } = new("McpCallFailed"); + + /// Gets a message type indicating that listing MCP tools is in progress. + public static RealtimeServerMessageType McpListToolsInProgress { get; } = new("McpListToolsInProgress"); + + /// Gets a message type indicating that listing MCP tools has completed. + public static RealtimeServerMessageType McpListToolsCompleted { get; } = new("McpListToolsCompleted"); + + /// Gets a message type indicating that listing MCP tools has failed. + public static RealtimeServerMessageType McpListToolsFailed { get; } = new("McpListToolsFailed"); /// - /// Indicates that an MCP tool call has failed. + /// Gets the value associated with this . /// - McpCallFailed, + public string Value { get; } /// - /// Indicates that listing MCP tools is in progress. + /// Initializes a new instance of the struct with the provided value. /// - McpListToolsInProgress, + /// The value to associate with this . + [JsonConstructor] + public RealtimeServerMessageType(string value) + { + Value = Throw.IfNullOrWhitespace(value); + } /// - /// Indicates that listing MCP tools has completed. + /// Returns a value indicating whether two instances are equivalent, as determined by a + /// case-insensitive comparison of their values. /// - McpListToolsCompleted, + /// The first instance to compare. + /// The second instance to compare. + /// if left and right have equivalent values; otherwise, . + public static bool operator ==(RealtimeServerMessageType left, RealtimeServerMessageType right) + { + return left.Equals(right); + } /// - /// Indicates that listing MCP tools has failed. + /// Returns a value indicating whether two instances are not equivalent, as determined by a + /// case-insensitive comparison of their values. /// - McpListToolsFailed, + /// The first instance to compare. + /// The second instance to compare. + /// if left and right have different values; otherwise, . + public static bool operator !=(RealtimeServerMessageType left, RealtimeServerMessageType right) + { + return !(left == right); + } + + /// + public override bool Equals([NotNullWhen(true)] object? obj) + => obj is RealtimeServerMessageType other && Equals(other); + + /// + public bool Equals(RealtimeServerMessageType other) + => string.Equals(Value, other.Value, StringComparison.OrdinalIgnoreCase); + + /// + public override int GetHashCode() + => Value is null ? 0 : StringComparer.OrdinalIgnoreCase.GetHashCode(Value); + + /// + public override string ToString() => Value ?? string.Empty; + + /// Provides a for serializing instances. + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class Converter : JsonConverter + { + /// + public override RealtimeServerMessageType Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + new(reader.GetString()!); + + /// + public override void Write(Utf8JsonWriter writer, RealtimeServerMessageType value, JsonSerializerOptions options) => + Throw.IfNull(writer).WriteStringValue(value.Value); + } } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index fa8ed439a5f..5da44e4138c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -1664,7 +1664,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (root.TryGetProperty("delta", out var deltaElement)) { - if (serverMessageType is RealtimeServerMessageType.OutputAudioDelta) + if (serverMessageType == RealtimeServerMessageType.OutputAudioDelta) { msg.Audio = deltaElement.GetString(); } diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs index b49e90cc4b5..7f4057b3894 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs @@ -364,14 +364,30 @@ private static void AddOutputMessagesTag(Activity? activity, ListGets the output modality from a server message, if applicable. private static string? GetOutputModality(RealtimeServerMessage message) { - return message switch + if (message is RealtimeServerOutputTextAudioMessage textAudio) { - RealtimeServerOutputTextAudioMessage { Type: RealtimeServerMessageType.OutputTextDelta or RealtimeServerMessageType.OutputTextDone } => "text", - RealtimeServerOutputTextAudioMessage { Type: RealtimeServerMessageType.OutputAudioDelta or RealtimeServerMessageType.OutputAudioDone } => "audio", - RealtimeServerOutputTextAudioMessage { Type: RealtimeServerMessageType.OutputAudioTranscriptionDelta or RealtimeServerMessageType.OutputAudioTranscriptionDone } => "transcription", - RealtimeServerResponseOutputItemMessage => "item", - _ => null, - }; + if (textAudio.Type == RealtimeServerMessageType.OutputTextDelta || textAudio.Type == RealtimeServerMessageType.OutputTextDone) + { + return "text"; + } + + if (textAudio.Type == RealtimeServerMessageType.OutputAudioDelta || textAudio.Type == RealtimeServerMessageType.OutputAudioDone) + { + return "audio"; + } + + if (textAudio.Type == RealtimeServerMessageType.OutputAudioTranscriptionDelta || textAudio.Type == RealtimeServerMessageType.OutputAudioTranscriptionDone) + { + return "transcription"; + } + } + + if (message is RealtimeServerResponseOutputItemMessage) + { + return "item"; + } + + return null; } /// Extracts an OTel message from a realtime client message. @@ -445,17 +461,24 @@ private static void AddOutputMessagesTag(Activity? activity, List - ("text", textAudioMsg.Text), - RealtimeServerMessageType.OutputAudioDelta or RealtimeServerMessageType.OutputAudioDone => - ("audio", string.IsNullOrEmpty(textAudioMsg.Audio) ? "[audio data]" : textAudioMsg.Audio), - RealtimeServerMessageType.OutputAudioTranscriptionDelta or RealtimeServerMessageType.OutputAudioTranscriptionDone => - ("output_transcription", textAudioMsg.Text), - _ => ("text", textAudioMsg.Text), - }; + partType = "audio"; + content = string.IsNullOrEmpty(textAudioMsg.Audio) ? "[audio data]" : textAudioMsg.Audio; + } + else if (textAudioMsg.Type == RealtimeServerMessageType.OutputAudioTranscriptionDelta || textAudioMsg.Type == RealtimeServerMessageType.OutputAudioTranscriptionDone) + { + partType = "output_transcription"; + content = textAudioMsg.Text; + } + else + { + partType = "text"; + content = textAudioMsg.Text; + } // Skip if no meaningful content if (string.IsNullOrEmpty(content)) From 156b3764a3d3c7210e4395cb81e98e1c352a94bc Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 13:40:44 -0800 Subject: [PATCH 27/92] Move Parameter into ErrorContent.Details on error messages Remove the Parameter property from RealtimeServerErrorMessage and map error.param to ErrorContent.Details instead. Improve ErrorEventId XML docs to clarify it correlates to the originating client event. --- .../Realtime/RealtimeServerErrorMessage.cs | 11 +++++------ .../OpenAIRealtimeSession.cs | 2 +- .../Realtime/RealtimeServerMessageTests.cs | 6 ++---- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs index 85ce29d73a4..e785318e9c8 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs @@ -29,13 +29,12 @@ public RealtimeServerErrorMessage() public ErrorContent? Error { get; set; } /// - /// Gets or sets an optional event ID caused the error. + /// Gets or sets the event ID of the client event that caused the error. /// + /// + /// This is specific to event-driven protocols where multiple client events may be in-flight, + /// allowing correlation of the error to the originating client request. + /// public string? ErrorEventId { get; set; } - /// - /// Gets or sets an optional parameter providing additional context about the error. - /// - public string? Parameter { get; set; } - } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 5da44e4138c..2dca1fa54c6 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -1555,7 +1555,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (errorElement.TryGetProperty("param", out var paramElement)) { - msg.Parameter = paramElement.GetString(); + msg.Error.Details = paramElement.GetString(); } return msg; diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs index 06e1b155559..8d327927b0e 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs @@ -51,24 +51,22 @@ public void ErrorMessage_DefaultProperties() Assert.Null(message.Error); Assert.Null(message.ErrorEventId); - Assert.Null(message.Parameter); } [Fact] public void ErrorMessage_Properties_Roundtrip() { - var error = new ErrorContent("Test error"); + var error = new ErrorContent("Test error") { Details = "temperature" }; var message = new RealtimeServerErrorMessage { Error = error, ErrorEventId = "evt_bad", - Parameter = "temperature", EventId = "evt_err_1", }; Assert.Same(error, message.Error); Assert.Equal("evt_bad", message.ErrorEventId); - Assert.Equal("temperature", message.Parameter); + Assert.Equal("temperature", message.Error.Details); Assert.Equal("evt_err_1", message.EventId); Assert.IsAssignableFrom(message); } From f59afd691c6912565262fc46e05c378fb872c408 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 13:49:17 -0800 Subject: [PATCH 28/92] Add RawRepresentation property to RealtimeContentItem Add object? RawRepresentation to hold the original provider data structure, following the same pattern as other types in the abstraction layer (e.g., ChatMessage). Updated tests accordingly. --- .../Realtime/RealtimeContentItem.cs | 6 ++++++ .../Realtime/RealtimeContentItemTests.cs | 3 +++ 2 files changed, 9 insertions(+) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs index 1aaf4e0721f..9f3040dc326 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs @@ -52,4 +52,10 @@ public RealtimeContentItem(IList contents, string? id = null, ChatRol /// Gets or sets the content of the conversation item. /// public IList Contents { get; set; } + + /// + /// Gets or sets the raw representation of the conversation item. + /// This can be used to hold the original data structure received from or sent to the provider. + /// + public object? RawRepresentation { get; set; } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeContentItemTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeContentItemTests.cs index b27a42c2bc8..c2e50936894 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeContentItemTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeContentItemTests.cs @@ -19,6 +19,7 @@ public void Constructor_WithContentsOnly_PropsDefaulted() Assert.Same(contents, item.Contents); Assert.Null(item.Id); Assert.Null(item.Role); + Assert.Null(item.RawRepresentation); } [Fact] @@ -42,10 +43,12 @@ public void Properties_Roundtrip() item.Id = "new_id"; item.Role = ChatRole.Assistant; item.Contents = newContents; + item.RawRepresentation = "raw_data"; Assert.Equal("new_id", item.Id); Assert.Equal(ChatRole.Assistant, item.Role); Assert.Same(newContents, item.Contents); + Assert.Equal("raw_data", item.RawRepresentation); } [Fact] From e8393c144ee369ea0a7faf8ba00b48cbcb815fab Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 14:03:49 -0800 Subject: [PATCH 29/92] Rename Metadata to AdditionalProperties for consistency Rename the Metadata property to AdditionalProperties on both RealtimeClientResponseCreateMessage and RealtimeServerResponseCreatedMessage to be consistent with the established pattern used across the AI abstractions (ChatMessage, ChatOptions, AIContent, etc.). Updated XML docs, OpenAI provider, OTel session, and tests accordingly. --- .../Realtime/RealtimeClientResponseCreateMessage.cs | 9 +++++++-- .../Realtime/RealtimeServerResponseCreatedMessage.cs | 9 +++++++-- .../OpenAIRealtimeSession.cs | 6 +++--- .../Realtime/OpenTelemetryRealtimeSession.cs | 2 +- .../Realtime/RealtimeClientMessageTests.cs | 6 +++--- .../Realtime/RealtimeServerMessageTests.cs | 6 +++--- 6 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs index a840195b4fd..5b6c8703018 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs @@ -52,9 +52,14 @@ public RealtimeClientResponseCreateMessage() public int? MaxOutputTokens { get; set; } /// - /// Gets or sets additional metadata for the message. + /// Gets or sets any additional properties associated with the response request. /// - public AdditionalPropertiesDictionary? Metadata { get; set; } + /// + /// This can be used to attach arbitrary key-value metadata to a response request + /// for tracking or disambiguation purposes (e.g., correlating multiple simultaneous responses). + /// Providers may map this to their own metadata fields. + /// + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } /// /// Gets or sets the output modalities for the response. like "text", "audio". diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs index dcac8956737..09d62b582e0 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs @@ -59,9 +59,14 @@ public RealtimeServerResponseCreatedMessage(RealtimeServerMessageType type) public int? MaxOutputTokens { get; set; } /// - /// Gets or sets additional metadata for the message. + /// Gets or sets any additional properties associated with the response. /// - public AdditionalPropertiesDictionary? Metadata { get; set; } + /// + /// Contains arbitrary key-value metadata attached to the response. + /// This is the metadata that was provided when the response was created + /// (e.g., for tracking or disambiguating multiple simultaneous responses). + /// + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } /// /// Gets or sets the list of the conversation items included in the response. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 2dca1fa54c6..f6257930f46 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -384,10 +384,10 @@ public async Task InjectClientMessageAsync(RealtimeClientMessage message, Cancel responseObj["max_output_tokens"] = responseCreate.MaxOutputTokens.Value; } - if (responseCreate.Metadata is { Count: > 0 }) + if (responseCreate.AdditionalProperties is { Count: > 0 }) { var metadataObj = new JsonObject(); - foreach (var kvp in responseCreate.Metadata) + foreach (var kvp in responseCreate.AdditionalProperties) { metadataObj[kvp.Key] = JsonValue.Create(kvp.Value); } @@ -1779,7 +1779,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess : JsonSerializer.Deserialize(property.Value.GetRawText()); } - msg.Metadata = metadataDict; + msg.AdditionalProperties = metadataDict; } if (responseElement.TryGetProperty("output_modalities", out var outputModalitiesElement)) diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs index 7f4057b3894..bcb76facb5a 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs @@ -964,7 +964,7 @@ private void TraceStreamingResponse( if (response is not null && activity is not null) { // Log metadata first so standard tags take precedence if keys collide - if (EnableSensitiveData && response.Metadata is { } metadata) + if (EnableSensitiveData && response.AdditionalProperties is { } metadata) { foreach (var prop in metadata) { diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs index a2d0530d39c..cc67e8347ea 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs @@ -137,7 +137,7 @@ public void ResponseCreateMessage_DefaultProperties() Assert.False(message.ExcludeFromConversation); Assert.Null(message.Instructions); Assert.Null(message.MaxOutputTokens); - Assert.Null(message.Metadata); + Assert.Null(message.AdditionalProperties); Assert.Null(message.OutputModalities); Assert.Null(message.ToolMode); Assert.Null(message.Tools); @@ -163,7 +163,7 @@ public void ResponseCreateMessage_Properties_Roundtrip() message.ExcludeFromConversation = true; message.Instructions = "Be brief"; message.MaxOutputTokens = 100; - message.Metadata = metadata; + message.AdditionalProperties = metadata; message.OutputModalities = modalities; message.ToolMode = ChatToolMode.Auto; message.Tools = tools; @@ -174,7 +174,7 @@ public void ResponseCreateMessage_Properties_Roundtrip() Assert.True(message.ExcludeFromConversation); Assert.Equal("Be brief", message.Instructions); Assert.Equal(100, message.MaxOutputTokens); - Assert.Same(metadata, message.Metadata); + Assert.Same(metadata, message.AdditionalProperties); Assert.Same(modalities, message.OutputModalities); Assert.Equal(ChatToolMode.Auto, message.ToolMode); Assert.Same(tools, message.Tools); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs index 8d327927b0e..15308933201 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs @@ -177,7 +177,7 @@ public void ResponseCreatedMessage_DefaultProperties() Assert.Null(message.ConversationId); Assert.Null(message.ResponseId); Assert.Null(message.MaxOutputTokens); - Assert.Null(message.Metadata); + Assert.Null(message.AdditionalProperties); Assert.Null(message.Items); Assert.Null(message.OutputModalities); Assert.Null(message.Status); @@ -205,7 +205,7 @@ public void ResponseCreatedMessage_Properties_Roundtrip() ConversationId = "conv_1", ResponseId = "resp_1", MaxOutputTokens = 1000, - Metadata = metadata, + AdditionalProperties = metadata, Items = items, OutputModalities = modalities, Status = "completed", @@ -218,7 +218,7 @@ public void ResponseCreatedMessage_Properties_Roundtrip() Assert.Equal("conv_1", message.ConversationId); Assert.Equal("resp_1", message.ResponseId); Assert.Equal(1000, message.MaxOutputTokens); - Assert.Same(metadata, message.Metadata); + Assert.Same(metadata, message.AdditionalProperties); Assert.Same(items, message.Items); Assert.Same(modalities, message.OutputModalities); Assert.Equal("completed", message.Status); From 6aae7d3d44962fa08e6f0b414fe479168abff637 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 14:11:49 -0800 Subject: [PATCH 30/92] Improve MaxOutputTokens XML docs to clarify modality scope Clarify that MaxOutputTokens is a total budget across all output modalities (text, audio) and tool calls, not per-modality. --- .../Realtime/RealtimeClientResponseCreateMessage.cs | 6 +++++- .../Realtime/RealtimeServerResponseCreatedMessage.cs | 7 +++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs index 5b6c8703018..b626ac950c9 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs @@ -47,8 +47,12 @@ public RealtimeClientResponseCreateMessage() public string? Instructions { get; set; } /// - /// Gets or sets the maximum number of output tokens for the response. + /// Gets or sets the maximum number of output tokens for the response, inclusive of all modalities and tool calls. /// + /// + /// This limit applies to the total output tokens regardless of modality (text, audio, etc.). + /// If , the provider's default limit is used. + /// public int? MaxOutputTokens { get; set; } /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs index 09d62b582e0..36823ae99e8 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs @@ -53,9 +53,12 @@ public RealtimeServerResponseCreatedMessage(RealtimeServerMessageType type) public string? ResponseId { get; set; } /// - /// Gets or sets the maximum number of output tokens for the response. - /// If 0, the service will apply its own limit. + /// Gets or sets the maximum number of output tokens for the response, inclusive of all modalities and tool calls. /// + /// + /// This limit applies to the total output tokens regardless of modality (text, audio, etc.). + /// If , the provider's default limit was used. + /// public int? MaxOutputTokens { get; set; } /// From c7f041b455baec35a2fff68b3881a57ae8f75522 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 15:08:14 -0800 Subject: [PATCH 31/92] Improve XML docs on RealtimeClientResponseCreateMessage properties Clarify that ExcludeFromConversation creates an out-of-band response whose output is not added to conversation history. Document that Instructions, Tools, ToolMode, OutputModalities, OutputAudioOptions, and OutputVoice are per-response overrides of session configuration. --- .../RealtimeClientResponseCreateMessage.cs | 40 ++++++++++++++++--- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs index b626ac950c9..2339ecb8f84 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs @@ -26,24 +26,41 @@ public RealtimeClientResponseCreateMessage() public IList? Items { get; set; } /// - /// Gets or sets the output audio options for the response. If null, the default conversation audio options will be used. + /// Gets or sets the output audio options for the response. /// + /// + /// If set, overrides the session-level audio output configuration for this response only. + /// If , the session's default audio options are used. + /// public RealtimeAudioFormat? OutputAudioOptions { get; set; } /// /// Gets or sets the voice of the output audio. /// + /// + /// If set, overrides the session-level voice for this response only. + /// If , the session's default voice is used. + /// public string? OutputVoice { get; set; } /// - /// Gets or sets a value indicating whether the response should be excluded from the conversation history. + /// Gets or sets a value indicating whether the response output should be excluded from the conversation context. /// + /// + /// When , the response is generated out-of-band: the model produces output + /// but the resulting items are not added to the conversation history, so they will not appear + /// as context for subsequent responses. Defaults to , meaning response + /// output is added to the default conversation. + /// public bool ExcludeFromConversation { get; set; } /// - /// Gets or sets the instructions allows the client to guide the model on desired responses. - /// If null, the default conversation instructions will be used. + /// Gets or sets the instructions that guide the model on desired responses. /// + /// + /// If set, overrides the session-level instructions for this response only. + /// If , the session's default instructions are used. + /// public string? Instructions { get; set; } /// @@ -66,18 +83,29 @@ public RealtimeClientResponseCreateMessage() public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } /// - /// Gets or sets the output modalities for the response. like "text", "audio". - /// If null, then default conversation modalities will be used. + /// Gets or sets the output modalities for the response (e.g., "text", "audio"). /// + /// + /// If set, overrides the session-level output modalities for this response only. + /// If , the session's default modalities are used. + /// public IList? OutputModalities { get; set; } /// /// Gets or sets the tool choice mode for the response. /// + /// + /// If set, overrides the session-level tool choice for this response only. + /// If , the session's default tool choice is used. + /// public ChatToolMode? ToolMode { get; set; } /// /// Gets or sets the AI tools available for generating the response. /// + /// + /// If set, overrides the session-level tools for this response only. + /// If , the session's default tools are used. + /// public IList? Tools { get; set; } } From cde880a6cf367bd951847179f434ed301ba86d59 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 15:18:49 -0800 Subject: [PATCH 32/92] Improve RealtimeClientResponseCreateMessage class-level XML doc Clarify that this message triggers model inference and that its properties are per-response overrides of session configuration. --- .../Realtime/RealtimeClientResponseCreateMessage.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs index 2339ecb8f84..bcd4a016f97 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs @@ -8,8 +8,14 @@ namespace Microsoft.Extensions.AI; /// -/// Represents a real-time message for creating a response item. +/// Represents a client message that triggers model inference to generate a response. /// +/// +/// Sending this message instructs the provider to generate a new response from the model. +/// The response may include one or more output items (text, audio, or tool calls). +/// Properties on this message optionally override the session-level configuration +/// for this response only. +/// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] public class RealtimeClientResponseCreateMessage : RealtimeClientMessage { From d2516c224713953f77f5c5029b3a02d7236a656f Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 15:33:49 -0800 Subject: [PATCH 33/92] Rename EventId to MessageId for terminology consistency Rename EventId to MessageId on RealtimeClientMessage and RealtimeServerMessage, and ErrorEventId to ErrorMessageId on RealtimeServerErrorMessage. The abstraction uses 'message' terminology throughout (class names, docs, method signatures), so properties should match. The OpenAI provider maps MessageId to/from the wire protocol's event_id field. --- .../Realtime/RealtimeClientMessage.cs | 4 +-- .../Realtime/RealtimeServerErrorMessage.cs | 6 ++-- .../Realtime/RealtimeServerMessage.cs | 4 +-- .../OpenAIRealtimeSession.cs | 28 +++++++++---------- .../Realtime/LoggingRealtimeSession.cs | 8 +++--- .../Realtime/RealtimeClientMessageTests.cs | 20 ++++++------- .../Realtime/RealtimeServerMessageTests.cs | 16 +++++------ .../DelegatingRealtimeSessionTests.cs | 4 +-- .../FunctionInvokingRealtimeSessionTests.cs | 12 ++++---- .../Realtime/LoggingRealtimeSessionTests.cs | 10 +++---- .../OpenTelemetryRealtimeSessionTests.cs | 2 +- .../Realtime/RealtimeSessionBuilderTests.cs | 4 +-- 12 files changed, 59 insertions(+), 59 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientMessage.cs index da4336d2314..0f035933462 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientMessage.cs @@ -13,10 +13,10 @@ namespace Microsoft.Extensions.AI; public class RealtimeClientMessage { /// - /// Gets or sets the optional event ID associated with the message. + /// Gets or sets the optional message ID associated with the message. /// This can be used for tracking and correlation purposes. /// - public string? EventId { get; set; } + public string? MessageId { get; set; } /// /// Gets or sets the raw representation of the message. diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs index e785318e9c8..ea3131482fa 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs @@ -29,12 +29,12 @@ public RealtimeServerErrorMessage() public ErrorContent? Error { get; set; } /// - /// Gets or sets the event ID of the client event that caused the error. + /// Gets or sets the message ID of the client message that caused the error. /// /// - /// This is specific to event-driven protocols where multiple client events may be in-flight, + /// This is specific to event-driven protocols where multiple client messages may be in-flight, /// allowing correlation of the error to the originating client request. /// - public string? ErrorEventId { get; set; } + public string? ErrorMessageId { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessage.cs index d2af4b4a99a..0e023fde4f4 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessage.cs @@ -18,10 +18,10 @@ public class RealtimeServerMessage public RealtimeServerMessageType Type { get; set; } /// - /// Gets or sets the optional event ID associated with the response. + /// Gets or sets the optional message ID associated with the response. /// This can be used for tracking and correlation purposes. /// - public string? EventId { get; set; } + public string? MessageId { get; set; } /// /// Gets or sets the raw representation of the response. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index f6257930f46..499d747bd7b 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -285,9 +285,9 @@ public async Task InjectClientMessageAsync(RealtimeClientMessage message, Cancel JsonObject? jsonMessage = new JsonObject(); - if (message.EventId is not null) + if (message.MessageId is not null) { - jsonMessage["event_id"] = message.EventId; + jsonMessage["event_id"] = message.MessageId; } switch (message) @@ -554,11 +554,11 @@ public async Task InjectClientMessageAsync(RealtimeClientMessage message, Cancel jsonMessage = rawJsonObject; } - // Preserve EventId if it was set on the message but not in the raw representation. - if (jsonMessage is not null && message.EventId is not null && + // Preserve MessageId if it was set on the message but not in the raw representation. + if (jsonMessage is not null && message.MessageId is not null && !jsonMessage.ContainsKey("event_id")) { - jsonMessage["event_id"] = message.EventId; + jsonMessage["event_id"] = message.MessageId; } break; @@ -1550,7 +1550,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (root.TryGetProperty("event_id", out var eventIdElement)) { - msg.EventId = eventIdElement.GetString(); + msg.MessageId = eventIdElement.GetString(); } if (errorElement.TryGetProperty("param", out var paramElement)) @@ -1575,7 +1575,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (root.TryGetProperty("event_id", out var eventIdElement)) { - msg.EventId = eventIdElement.GetString(); + msg.MessageId = eventIdElement.GetString(); } if (root.TryGetProperty("content_index", out var contentIndexElement)) @@ -1639,7 +1639,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (root.TryGetProperty("event_id", out var eventIdElement)) { - msg.EventId = eventIdElement.GetString(); + msg.MessageId = eventIdElement.GetString(); } if (root.TryGetProperty("response_id", out var responseIdElement)) @@ -1695,7 +1695,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (root.TryGetProperty("event_id", out var eventIdElement)) { - msg.EventId = eventIdElement.GetString(); + msg.MessageId = eventIdElement.GetString(); } if (root.TryGetProperty("response_id", out var responseIdElement)) @@ -1734,7 +1734,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (root.TryGetProperty("event_id", out var eventIdElement)) { - msg.EventId = eventIdElement.GetString(); + msg.MessageId = eventIdElement.GetString(); } if (responseElement.TryGetProperty("audio", out var responseAudioElement) && @@ -1845,7 +1845,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess if (root.TryGetProperty("event_id", out var eventIdElement)) { - msg.EventId = eventIdElement.GetString(); + msg.MessageId = eventIdElement.GetString(); } if (root.TryGetProperty("response_id", out var responseIdElement)) @@ -1884,7 +1884,7 @@ private static RealtimeServerResponseOutputItemMessage CreateMcpListToolsMessage var msg = new RealtimeServerResponseOutputItemMessage(serverMessageType) { RawRepresentation = root.Clone(), - EventId = root.TryGetProperty("event_id", out var eventIdElement) ? eventIdElement.GetString() : null, + MessageId = root.TryGetProperty("event_id", out var eventIdElement) ? eventIdElement.GetString() : null, }; string? itemId = root.TryGetProperty("item_id", out var itemIdElement) ? itemIdElement.GetString() : null; @@ -1949,7 +1949,7 @@ private static RealtimeServerResponseOutputItemMessage CreateMcpListToolsMessage return new RealtimeServerResponseOutputItemMessage(RealtimeServerMessageType.RawContentOnly) { RawRepresentation = root.Clone(), - EventId = root.TryGetProperty("event_id", out var evtElement) ? evtElement.GetString() : null, + MessageId = root.TryGetProperty("event_id", out var evtElement) ? evtElement.GetString() : null, }; } @@ -1961,7 +1961,7 @@ private static RealtimeServerResponseOutputItemMessage CreateMcpListToolsMessage if (root.TryGetProperty("event_id", out var eventIdElement)) { - msg.EventId = eventIdElement.GetString(); + msg.MessageId = eventIdElement.GetString(); } return msg; diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs index a61ed689ec3..520b0748b09 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs @@ -217,9 +217,9 @@ private string GetLoggableString(RealtimeClientMessage message) { obj["content"] = AsJson(message.RawRepresentation); } - else if (message.EventId is not null) + else if (message.MessageId is not null) { - obj["eventId"] = message.EventId; + obj["messageId"] = message.MessageId; } return obj.ToJsonString(); @@ -240,9 +240,9 @@ private string GetLoggableString(RealtimeServerMessage message) { obj["content"] = AsJson(message.RawRepresentation); } - else if (message.EventId is not null) + else if (message.MessageId is not null) { - obj["eventId"] = message.EventId; + obj["messageId"] = message.MessageId; } return obj.ToJsonString(); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs index cc67e8347ea..f97a521f083 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs @@ -16,7 +16,7 @@ public void RealtimeClientMessage_DefaultProperties() { var message = new RealtimeClientMessage(); - Assert.Null(message.EventId); + Assert.Null(message.MessageId); Assert.Null(message.RawRepresentation); } @@ -26,11 +26,11 @@ public void RealtimeClientMessage_Properties_Roundtrip() var rawObj = new object(); var message = new RealtimeClientMessage { - EventId = "evt_001", + MessageId = "evt_001", RawRepresentation = rawObj, }; - Assert.Equal("evt_001", message.EventId); + Assert.Equal("evt_001", message.MessageId); Assert.Same(rawObj, message.RawRepresentation); } @@ -76,10 +76,10 @@ public void ConversationItemCreateMessage_InheritsClientMessage() var item = new RealtimeContentItem([new TextContent("Hello")]); var message = new RealtimeClientConversationItemCreateMessage(item) { - EventId = "evt_create_1", + MessageId = "evt_create_1", }; - Assert.Equal("evt_create_1", message.EventId); + Assert.Equal("evt_create_1", message.MessageId); Assert.IsAssignableFrom(message); } @@ -110,10 +110,10 @@ public void InputAudioBufferAppendMessage_InheritsClientMessage() var audioContent = new DataContent(new byte[] { 1, 2, 3 }, "audio/pcm"); var message = new RealtimeClientInputAudioBufferAppendMessage(audioContent) { - EventId = "evt_append_1", + MessageId = "evt_append_1", }; - Assert.Equal("evt_append_1", message.EventId); + Assert.Equal("evt_append_1", message.MessageId); Assert.IsAssignableFrom(message); } @@ -123,7 +123,7 @@ public void InputAudioBufferCommitMessage_Constructor() var message = new RealtimeClientInputAudioBufferCommitMessage(); Assert.IsAssignableFrom(message); - Assert.Null(message.EventId); + Assert.Null(message.MessageId); } [Fact] @@ -185,11 +185,11 @@ public void ResponseCreateMessage_InheritsClientMessage() { var message = new RealtimeClientResponseCreateMessage { - EventId = "evt_resp_1", + MessageId = "evt_resp_1", RawRepresentation = "raw", }; - Assert.Equal("evt_resp_1", message.EventId); + Assert.Equal("evt_resp_1", message.MessageId); Assert.Equal("raw", message.RawRepresentation); Assert.IsAssignableFrom(message); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs index 15308933201..fe1c1ef9837 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs @@ -16,7 +16,7 @@ public void RealtimeServerMessage_DefaultProperties() var message = new RealtimeServerMessage(); Assert.Equal(default, message.Type); - Assert.Null(message.EventId); + Assert.Null(message.MessageId); Assert.Null(message.RawRepresentation); } @@ -27,12 +27,12 @@ public void RealtimeServerMessage_Properties_Roundtrip() var message = new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseDone, - EventId = "evt_001", + MessageId = "evt_001", RawRepresentation = rawObj, }; Assert.Equal(RealtimeServerMessageType.ResponseDone, message.Type); - Assert.Equal("evt_001", message.EventId); + Assert.Equal("evt_001", message.MessageId); Assert.Same(rawObj, message.RawRepresentation); } @@ -50,7 +50,7 @@ public void ErrorMessage_DefaultProperties() var message = new RealtimeServerErrorMessage(); Assert.Null(message.Error); - Assert.Null(message.ErrorEventId); + Assert.Null(message.ErrorMessageId); } [Fact] @@ -60,14 +60,14 @@ public void ErrorMessage_Properties_Roundtrip() var message = new RealtimeServerErrorMessage { Error = error, - ErrorEventId = "evt_bad", - EventId = "evt_err_1", + ErrorMessageId = "evt_bad", + MessageId = "evt_err_1", }; Assert.Same(error, message.Error); - Assert.Equal("evt_bad", message.ErrorEventId); + Assert.Equal("evt_bad", message.ErrorMessageId); Assert.Equal("temperature", message.Error.Details); - Assert.Equal("evt_err_1", message.EventId); + Assert.Equal("evt_err_1", message.MessageId); Assert.IsAssignableFrom(message); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs index 617fe1f0fb9..a9f6df9b410 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs @@ -54,7 +54,7 @@ public async Task UpdateAsync_DelegatesToInner() public async Task InjectClientMessageAsync_DelegatesToInner() { var called = false; - var sentMessage = new RealtimeClientMessage { EventId = "evt_001" }; + var sentMessage = new RealtimeClientMessage { MessageId = "evt_001" }; using var inner = new TestRealtimeSession { InjectClientMessageAsyncCallback = (msg, _) => @@ -73,7 +73,7 @@ public async Task InjectClientMessageAsync_DelegatesToInner() [Fact] public async Task GetStreamingResponseAsync_DelegatesToInner() { - var expected = new RealtimeServerMessage { Type = RealtimeServerMessageType.Error, EventId = "evt_002" }; + var expected = new RealtimeServerMessage { Type = RealtimeServerMessageType.Error, MessageId = "evt_002" }; using var inner = new TestRealtimeSession { GetStreamingResponseAsyncCallback = (_, ct) => YieldSingle(expected, ct), diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs index 554349ad4c8..fb953217bc3 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs @@ -81,8 +81,8 @@ public async Task GetStreamingResponseAsync_NoFunctionCalls_PassesThrough() { var serverMessages = new RealtimeServerMessage[] { - new() { Type = RealtimeServerMessageType.ResponseCreated, EventId = "evt_001" }, - new() { Type = RealtimeServerMessageType.ResponseDone, EventId = "evt_002" }, + new() { Type = RealtimeServerMessageType.ResponseCreated, MessageId = "evt_001" }, + new() { Type = RealtimeServerMessageType.ResponseDone, MessageId = "evt_002" }, }; using var inner = new TestRealtimeSession @@ -98,8 +98,8 @@ public async Task GetStreamingResponseAsync_NoFunctionCalls_PassesThrough() } Assert.Equal(2, received.Count); - Assert.Equal("evt_001", received[0].EventId); - Assert.Equal("evt_002", received[1].EventId); + Assert.Equal("evt_001", received[0].MessageId); + Assert.Equal("evt_002", received[1].MessageId); } [Fact] @@ -395,7 +395,7 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_StopsLoop() GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_unknown", "nonexistent_func", null), - new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseDone, EventId = "should_not_reach" }, + new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseDone, MessageId = "should_not_reach" }, ], ct), InjectClientMessageAsyncCallback = (msg, _) => { @@ -621,7 +621,7 @@ public async Task GetStreamingResponseAsync_NonInvocableTool_TerminatesLoop() GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_decl", "my_declaration", null), - new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseDone, EventId = "should_not_reach" }, + new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseDone, MessageId = "should_not_reach" }, ], ct), InjectClientMessageAsyncCallback = (msg, _) => { diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs index 52672dd7689..5a26dc339d8 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs @@ -108,7 +108,7 @@ public async Task InjectClientMessageAsync_LogsInvocationAndCompletion(LogLevel .UseLogging() .Build(services); - await session.InjectClientMessageAsync(new RealtimeClientMessage { EventId = "test-event-123" }); + await session.InjectClientMessageAsync(new RealtimeClientMessage { MessageId = "test-event-123" }); var logs = collector.GetSnapshot(); if (level is LogLevel.Trace) @@ -146,8 +146,8 @@ public async Task GetStreamingResponseAsync_LogsMessagesReceived(LogLevel level) static async IAsyncEnumerable GetMessagesAsync() { await Task.Yield(); - yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.OutputTextDelta, EventId = "event-1" }; - yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.OutputAudioDelta, EventId = "event-2" }; + yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.OutputTextDelta, MessageId = "event-1" }; + yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.OutputAudioDelta, MessageId = "event-2" }; } using var session = innerSession @@ -360,7 +360,7 @@ public async Task InjectClientMessageAsync_LogsCancellation() cts.Cancel(); await Assert.ThrowsAsync(() => - session.InjectClientMessageAsync(new RealtimeClientMessage { EventId = "evt_cancel" }, cts.Token)); + session.InjectClientMessageAsync(new RealtimeClientMessage { MessageId = "evt_cancel" }, cts.Token)); var logs = collector.GetSnapshot(); Assert.Collection(logs, @@ -511,6 +511,6 @@ private static async IAsyncEnumerable GetClientMessages( { _ = cancellationToken; await Task.CompletedTask.ConfigureAwait(false); - yield return new RealtimeClientMessage { EventId = "client_evt_1" }; + yield return new RealtimeClientMessage { MessageId = "client_evt_1" }; } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs index 6111ee0fe8d..79552329be4 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs @@ -58,7 +58,7 @@ static async IAsyncEnumerable CallbackAsync( // Just consume the update } - yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseCreated, EventId = "evt_001" }; + yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseCreated, MessageId = "evt_001" }; yield return new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputTextDelta) { OutputIndex = 0, Text = "Hello" }; yield return new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputTextDelta) { OutputIndex = 0, Text = " there!" }; yield return new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputTextDone) { OutputIndex = 0, Text = "Hello there!" }; diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs index 997d28aefcf..25b10374d75 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs @@ -142,7 +142,7 @@ public async Task Use_WithStreamingDelegate_InterceptsStreaming() var intercepted = false; using var inner = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (_, ct) => YieldSingle(new RealtimeServerMessage { EventId = "inner" }, ct), + GetStreamingResponseAsyncCallback = (_, ct) => YieldSingle(new RealtimeServerMessage { MessageId = "inner" }, ct), }; var builder = new RealtimeSessionBuilder(inner); @@ -155,7 +155,7 @@ public async Task Use_WithStreamingDelegate_InterceptsStreaming() using var pipeline = builder.Build(); await foreach (var msg in pipeline.GetStreamingResponseAsync(EmptyUpdates())) { - Assert.Equal("inner", msg.EventId); + Assert.Equal("inner", msg.MessageId); } Assert.True(intercepted); From 1d15400439c1b2e222e49125480988e3b76ddc4c Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 15:46:35 -0800 Subject: [PATCH 34/92] Remove blank line between using directives in audio buffer messages --- .../Realtime/RealtimeClientInputAudioBufferAppendMessage.cs | 1 - .../Realtime/RealtimeClientInputAudioBufferCommitMessage.cs | 1 - 2 files changed, 2 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs index 86c54816a89..2e1f9c998d2 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics.CodeAnalysis; - using Microsoft.Shared.DiagnosticIds; namespace Microsoft.Extensions.AI; diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs index 8415588525f..15be87316d3 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics.CodeAnalysis; - using Microsoft.Shared.DiagnosticIds; namespace Microsoft.Extensions.AI; From 0ca0a172b0807b39aa159f5d130bc652e82d12b0 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Sun, 22 Feb 2026 16:02:52 -0800 Subject: [PATCH 35/92] Rename RealtimeAudioFormat.Type to MediaType for consistency Rename to match the established MediaType naming convention used across the abstractions (DataContent, HostedFileContent, UriContent, ImageGenerationOptions). Updated OpenAI provider and tests. --- .../Realtime/RealtimeAudioFormat.cs | 8 ++++---- .../OpenAIRealtimeSession.cs | 10 +++++----- .../Realtime/RealtimeAudioFormatTests.cs | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs index 37535d634d2..93c97d08fd2 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs @@ -15,16 +15,16 @@ public class RealtimeAudioFormat /// /// Initializes a new instance of the class. /// - public RealtimeAudioFormat(string type, int sampleRate) + public RealtimeAudioFormat(string mediaType, int sampleRate) { - Type = type; + MediaType = mediaType; SampleRate = sampleRate; } /// - /// Gets or sets the type of audio. For example, "audio/pcm". + /// Gets or sets the media type of the audio (e.g., "audio/pcm", "audio/pcmu", "audio/pcma"). /// - public string Type { get; set; } + public string MediaType { get; set; } /// /// Gets or sets the sample rate of the audio in Hertz. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 499d747bd7b..ffb1cd88a30 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -140,7 +140,7 @@ public async Task UpdateAsync(RealtimeSessionOptions options, CancellationToken { var audioInputFormatElement = new JsonObject { - ["type"] = options.InputAudioFormat.Type, + ["type"] = options.InputAudioFormat.MediaType, }; if (options.InputAudioFormat.SampleRate.HasValue) { @@ -206,7 +206,7 @@ public async Task UpdateAsync(RealtimeSessionOptions options, CancellationToken { var audioOutputFormatElement = new JsonObject { - ["type"] = options.OutputAudioFormat.Type, + ["type"] = options.OutputAudioFormat.MediaType, }; if (options.OutputAudioFormat.SampleRate.HasValue) { @@ -303,12 +303,12 @@ public async Task InjectClientMessageAsync(RealtimeClientMessage message, Cancel var outputObj = new JsonObject(); var formatObj = new JsonObject(); - switch (responseCreate.OutputAudioOptions.Type) + switch (responseCreate.OutputAudioOptions.MediaType) { case "audio/pcm": if (responseCreate.OutputAudioOptions.SampleRate == 24000) { - formatObj["type"] = responseCreate.OutputAudioOptions.Type; + formatObj["type"] = responseCreate.OutputAudioOptions.MediaType; formatObj["rate"] = 24000; } @@ -316,7 +316,7 @@ public async Task InjectClientMessageAsync(RealtimeClientMessage message, Cancel case "audio/pcmu": case "audio/pcma": - formatObj["type"] = responseCreate.OutputAudioOptions.Type; + formatObj["type"] = responseCreate.OutputAudioOptions.MediaType; break; } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeAudioFormatTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeAudioFormatTests.cs index 181bb5a64ed..e6af6ba6b97 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeAudioFormatTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeAudioFormatTests.cs @@ -14,7 +14,7 @@ public void Constructor_SetsProperties() { var format = new RealtimeAudioFormat("audio/pcm", 16000); - Assert.Equal("audio/pcm", format.Type); + Assert.Equal("audio/pcm", format.MediaType); Assert.Equal(16000, format.SampleRate); } @@ -23,11 +23,11 @@ public void Properties_Roundtrip() { var format = new RealtimeAudioFormat("audio/pcm", 16000) { - Type = "audio/wav", + MediaType = "audio/wav", SampleRate = 24000, }; - Assert.Equal("audio/wav", format.Type); + Assert.Equal("audio/wav", format.MediaType); Assert.Equal(24000, format.SampleRate); } From 03014ce34d95c17ed4e0ff5e80ec05f646303f04 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Mon, 23 Feb 2026 11:02:39 -0800 Subject: [PATCH 36/92] Refactor IRealtimeSession: rename InjectClientMessageAsync to SendClientMessageAsync and remove updates parameter from GetStreamingResponseAsync - Rename InjectClientMessageAsync -> SendClientMessageAsync across all implementations - Remove IAsyncEnumerable updates parameter from GetStreamingResponseAsync - Move per-message telemetry from WrapClientMessagesForTelemetryAsync into SendClientMessageAsync override in OpenTelemetryRealtimeSession - Delete WrapUpdatesWithLoggingAsync from LoggingRealtimeSession - Delete WrapClientMessagesForTelemetryAsync from OpenTelemetryRealtimeSession - Update AnonymousDelegatingRealtimeSession delegate signature - Update RealtimeSessionBuilder.Use overload signature - Update all tests to use new API Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Realtime/DelegatingRealtimeSession.cs | 8 +- .../Realtime/IRealtimeSession.cs | 15 +- .../OpenAIRealtimeSession.cs | 30 +- .../AnonymousDelegatingRealtimeSession.cs | 10 +- .../FunctionInvokingRealtimeSession.cs | 8 +- .../Realtime/LoggingRealtimeSession.cs | 56 +--- .../Realtime/OpenTelemetryRealtimeSession.cs | 78 ++--- .../Realtime/RealtimeSessionBuilder.cs | 6 +- .../TestRealtimeSession.cs | 14 +- .../OpenAIRealtimeSessionTests.cs | 22 +- .../DelegatingRealtimeSessionTests.cs | 28 +- .../FunctionInvokingRealtimeSessionTests.cs | 99 +++--- .../Realtime/LoggingRealtimeSessionTests.cs | 117 ++----- .../OpenTelemetryRealtimeSessionTests.cs | 313 ++++++++++-------- .../Realtime/RealtimeSessionBuilderTests.cs | 21 +- 15 files changed, 327 insertions(+), 498 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs index e9476b2aefe..4bb1defb3b9 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs @@ -68,8 +68,8 @@ protected virtual async ValueTask DisposeAsyncCore() public virtual RealtimeSessionOptions? Options => InnerSession.Options; /// - public virtual Task InjectClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => - InnerSession.InjectClientMessageAsync(message, cancellationToken); + public virtual Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => + InnerSession.SendClientMessageAsync(message, cancellationToken); /// public virtual Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancellationToken = default) => @@ -77,8 +77,8 @@ public virtual Task UpdateAsync(RealtimeSessionOptions options, CancellationToke /// public virtual IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, CancellationToken cancellationToken = default) => - InnerSession.GetStreamingResponseAsync(updates, cancellationToken); + CancellationToken cancellationToken = default) => + InnerSession.GetStreamingResponseAsync(cancellationToken); /// public virtual object? GetService(Type serviceType, object? serviceKey = null) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs index 0d1f6a05a8a..ae27baf91b1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs @@ -27,25 +27,24 @@ public interface IRealtimeSession : IDisposable, IAsyncDisposable RealtimeSessionOptions? Options { get; } /// - /// Injects a client message into the session. + /// Sends a client message to the session. /// - /// The client message to inject. + /// The client message to send. /// A token to cancel the operation. - /// A task that represents the asynchronous injection operation. + /// A task that represents the asynchronous send operation. /// - /// This method allows for the injection of client messages into the session at any time, which can be used to influence the session's behavior or state. + /// This method allows for sending client messages to the session at any time, which can be used to influence the session's behavior or state. /// - Task InjectClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default); + Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default); - /// Sends real-time messages and streams the response. - /// The sequence of real-time messages to send. + /// Streams the response from the real-time session. /// A token to cancel the operation. /// The response messages generated by the session. /// /// This method cannot be called multiple times concurrently on the same session instance. /// IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, CancellationToken cancellationToken = default); + CancellationToken cancellationToken = default); /// Asks the for an object of the specified type . /// The type of object being requested. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index ffb1cd88a30..e3662821608 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -271,7 +271,7 @@ public async Task UpdateAsync(RealtimeSessionOptions options, CancellationToken } /// - public async Task InjectClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) + public async Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) { _ = Throw.IfNull(message); @@ -572,40 +572,12 @@ public async Task InjectClientMessageAsync(RealtimeClientMessage message, Cancel /// public async IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - _ = Throw.IfNull(updates); - - var processUpdatesTask = Task.Run(async () => - { - try - { - await foreach (var message in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - await InjectClientMessageAsync(message, cancellationToken).ConfigureAwait(false); - } - } - catch (OperationCanceledException) - { - // Expected when the session is cancelled. - } - catch (ObjectDisposedException) - { - // Expected when the session is disposed concurrently. - } - catch (WebSocketException) - { - // Expected when the WebSocket is in an aborted state. - } - }, cancellationToken); - await foreach (var serverEvent in _eventChannel.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) { yield return serverEvent; } - - await processUpdatesTask.ConfigureAwait(false); } /// diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeSession.cs index a30c103ad4f..6566f0deb44 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeSession.cs @@ -14,7 +14,7 @@ namespace Microsoft.Extensions.AI; internal sealed class AnonymousDelegatingRealtimeSession : DelegatingRealtimeSession { /// The delegate to use as the implementation of . - private readonly Func, IRealtimeSession, CancellationToken, IAsyncEnumerable> _getStreamingResponseFunc; + private readonly Func> _getStreamingResponseFunc; /// /// Initializes a new instance of the class. @@ -27,7 +27,7 @@ internal sealed class AnonymousDelegatingRealtimeSession : DelegatingRealtimeSes /// is . public AnonymousDelegatingRealtimeSession( IRealtimeSession innerSession, - Func, IRealtimeSession, CancellationToken, IAsyncEnumerable> getStreamingResponseFunc) + Func> getStreamingResponseFunc) : base(innerSession) { _getStreamingResponseFunc = Throw.IfNull(getStreamingResponseFunc); @@ -35,10 +35,8 @@ public AnonymousDelegatingRealtimeSession( /// public override IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default) { - _ = Throw.IfNull(updates); - - return _getStreamingResponseFunc(updates, InnerSession, cancellationToken); + return _getStreamingResponseFunc(InnerSession, cancellationToken); } } diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs index 452a71bec98..87ca8f7efdf 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs @@ -250,10 +250,8 @@ public int MaximumConsecutiveErrorsPerRequest /// public override async IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken = default) + [EnumeratorCancellation] CancellationToken cancellationToken = default) { - _ = Throw.IfNull(updates); - // Create an activity to group function invocations together for better observability. using Activity? activity = FunctionInvocationHelpers.CurrentActivityIsInvokeAgent ? null : _activitySource?.StartActivity(OpenTelemetryConsts.GenAI.OrchestrateToolsName); @@ -262,7 +260,7 @@ public override async IAsyncEnumerable GetStreamingRespon int consecutiveErrorCount = 0; int iterationCount = 0; - await foreach (var message in InnerSession.GetStreamingResponseAsync(updates, cancellationToken).ConfigureAwait(false)) + await foreach (var message in InnerSession.GetStreamingResponseAsync(cancellationToken).ConfigureAwait(false)) { // Check if this message contains function calls bool hasFunctionCalls = false; @@ -307,7 +305,7 @@ public override async IAsyncEnumerable GetStreamingRespon foreach (var resultMessage in results.functionResults) { // inject back the function result messages to the inner session - await InnerSession.InjectClientMessageAsync(resultMessage, cancellationToken).ConfigureAwait(false); + await InnerSession.SendClientMessageAsync(resultMessage, cancellationToken).ConfigureAwait(false); } } } diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs index 520b0748b09..34ece9a8df9 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs @@ -90,7 +90,7 @@ public override async Task UpdateAsync(RealtimeSessionOptions options, Cancellat } /// - public override async Task InjectClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) + public override async Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) { _ = Throw.IfNull(message); @@ -98,41 +98,39 @@ public override async Task InjectClientMessageAsync(RealtimeClientMessage messag { if (_logger.IsEnabled(LogLevel.Trace)) { - LogInjectMessageSensitive(GetLoggableString(message)); + LogSendMessageSensitive(GetLoggableString(message)); } else { - LogInjectMessage(); + LogSendMessage(); } } try { - await base.InjectClientMessageAsync(message, cancellationToken).ConfigureAwait(false); + await base.SendClientMessageAsync(message, cancellationToken).ConfigureAwait(false); if (_logger.IsEnabled(LogLevel.Debug)) { - LogCompleted(nameof(InjectClientMessageAsync)); + LogCompleted(nameof(SendClientMessageAsync)); } } catch (OperationCanceledException) { - LogInvocationCanceled(nameof(InjectClientMessageAsync)); + LogInvocationCanceled(nameof(SendClientMessageAsync)); throw; } catch (Exception ex) { - LogInvocationFailed(nameof(InjectClientMessageAsync), ex); + LogInvocationFailed(nameof(SendClientMessageAsync), ex); throw; } } /// public override async IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken = default) + [EnumeratorCancellation] CancellationToken cancellationToken = default) { - _ = Throw.IfNull(updates); - if (_logger.IsEnabled(LogLevel.Debug)) { LogInvoked(nameof(GetStreamingResponseAsync)); @@ -141,7 +139,7 @@ public override async IAsyncEnumerable GetStreamingRespon IAsyncEnumerator e; try { - e = base.GetStreamingResponseAsync(WrapUpdatesWithLoggingAsync(updates, cancellationToken), cancellationToken).GetAsyncEnumerator(cancellationToken); + e = base.GetStreamingResponseAsync(cancellationToken).GetAsyncEnumerator(cancellationToken); } catch (OperationCanceledException) { @@ -248,28 +246,6 @@ private string GetLoggableString(RealtimeServerMessage message) return obj.ToJsonString(); } - private async IAsyncEnumerable WrapUpdatesWithLoggingAsync( - IAsyncEnumerable updates, - [EnumeratorCancellation] CancellationToken cancellationToken) - { - await foreach (var message in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - if (_logger.IsEnabled(LogLevel.Debug)) - { - if (_logger.IsEnabled(LogLevel.Trace)) - { - LogStreamingClientMessageSensitive(GetLoggableString(message)); - } - else - { - LogStreamingClientMessage(); - } - } - - yield return message; - } - } - private string AsJson(T value) => TelemetryHelpers.AsJson(value, _jsonSerializerOptions); [LoggerMessage(LogLevel.Debug, "{MethodName} invoked.")] @@ -278,21 +254,15 @@ private async IAsyncEnumerable WrapUpdatesWithLoggingAsyn [LoggerMessage(LogLevel.Trace, "{MethodName} invoked: Options: {Options}.")] private partial void LogInvokedSensitive(string methodName, string options); - [LoggerMessage(LogLevel.Debug, "InjectClientMessageAsync invoked.")] - private partial void LogInjectMessage(); + [LoggerMessage(LogLevel.Debug, "SendClientMessageAsync invoked.")] + private partial void LogSendMessage(); - [LoggerMessage(LogLevel.Trace, "InjectClientMessageAsync invoked: Message: {Message}.")] - private partial void LogInjectMessageSensitive(string message); + [LoggerMessage(LogLevel.Trace, "SendClientMessageAsync invoked: Message: {Message}.")] + private partial void LogSendMessageSensitive(string message); [LoggerMessage(LogLevel.Debug, "{MethodName} completed.")] private partial void LogCompleted(string methodName); - [LoggerMessage(LogLevel.Debug, "GetStreamingResponseAsync sending client message.")] - private partial void LogStreamingClientMessage(); - - [LoggerMessage(LogLevel.Trace, "GetStreamingResponseAsync sending client message: {ClientMessage}")] - private partial void LogStreamingClientMessageSensitive(string clientMessage); - [LoggerMessage(LogLevel.Debug, "GetStreamingResponseAsync received server message.")] private partial void LogStreamingServerMessage(); diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs index bcb76facb5a..316a107d9f2 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs @@ -199,11 +199,42 @@ public override async Task UpdateAsync(RealtimeSessionOptions options, Cancellat } } + /// + public override async Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) + { + if (EnableSensitiveData && _activitySource.HasListeners()) + { + var otelMessage = ExtractClientOtelMessage(message); + + if (otelMessage is not null) + { + RealtimeSessionOptions? options = Options; + string? requestModelId = options?.Model ?? _defaultModelId; + Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + + using Activity? inputActivity = CreateAndConfigureActivity(options: null); + if (inputActivity is { IsAllDataRequested: true }) + { + _ = inputActivity.AddTag(OpenTelemetryConsts.GenAI.Input.Messages, SerializeMessage(otelMessage)); + } + + // Record metrics + if (_operationDurationHistogram.Enabled && stopwatch is not null) + { + TagList tags = default; + AddMetricTags(ref tags, requestModelId, responseModelId: null); + _operationDurationHistogram.Record(stopwatch.Elapsed.TotalSeconds, tags); + } + } + } + + await base.SendClientMessageAsync(message, cancellationToken).ConfigureAwait(false); + } + /// public override async IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken = default) + [EnumeratorCancellation] CancellationToken cancellationToken = default) { - _ = Throw.IfNull(updates); _jsonSerializerOptions.MakeReadOnly(); RealtimeSessionOptions? options = Options; @@ -215,15 +246,10 @@ public override async IAsyncEnumerable GetStreamingRespon // Determine if we should capture messages for telemetry bool captureMessages = EnableSensitiveData && _activitySource.HasListeners(); - // Wrap client messages to capture input content and create input activity - IAsyncEnumerable wrappedUpdates = captureMessages - ? WrapClientMessagesForTelemetryAsync(updates, options, cancellationToken) - : updates; - IAsyncEnumerable responses; try { - responses = base.GetStreamingResponseAsync(wrappedUpdates, cancellationToken); + responses = base.GetStreamingResponseAsync(cancellationToken); } catch (Exception ex) { @@ -307,42 +333,6 @@ public override async IAsyncEnumerable GetStreamingRespon } } - /// Wraps client messages to capture content for telemetry with its own activity. - private async IAsyncEnumerable WrapClientMessagesForTelemetryAsync( - IAsyncEnumerable updates, - RealtimeSessionOptions? options, - [EnumeratorCancellation] CancellationToken cancellationToken) - { - Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; - string? requestModelId = options?.Model ?? _defaultModelId; - - await foreach (var message in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Capture input content from current client message - var otelMessage = ExtractClientOtelMessage(message); - - // Only create activity when there's content to log - if (otelMessage is not null) - { - using Activity? inputActivity = CreateAndConfigureActivity(options: null); - if (inputActivity is { IsAllDataRequested: true }) - { - _ = inputActivity.AddTag(OpenTelemetryConsts.GenAI.Input.Messages, SerializeMessage(otelMessage)); - } - - // Record metrics - if (_operationDurationHistogram.Enabled && stopwatch is not null) - { - TagList tags = default; - AddMetricTags(ref tags, requestModelId, responseModelId: null); - _operationDurationHistogram.Record(stopwatch.Elapsed.TotalSeconds, tags); - } - } - - yield return message; - } - } - /// Adds output modalities tag to the activity. private static void AddOutputModalitiesTag(Activity? activity, HashSet? outputModalities) { diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilder.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilder.cs index 545c68b9d77..de02b3dbe1d 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilder.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilder.cs @@ -92,8 +92,8 @@ public RealtimeSessionBuilder Use(Func /// /// A delegate that provides the implementation for . - /// This delegate is invoked with the sequence of realtime client messages, a delegate that represents invoking - /// the inner session, and a cancellation token. The delegate should be passed whatever client messages and + /// This delegate is invoked with a delegate that represents invoking + /// the inner session, and a cancellation token. The delegate should be passed whatever /// cancellation token should be passed along to the next stage in the pipeline. /// /// The updated instance. @@ -103,7 +103,7 @@ public RealtimeSessionBuilder Use(Func /// is . public RealtimeSessionBuilder Use( - Func, IRealtimeSession, CancellationToken, IAsyncEnumerable> getStreamingResponseFunc) + Func> getStreamingResponseFunc) { _ = Throw.IfNull(getStreamingResponseFunc); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs index 4dc48292694..4ec5d6e03a8 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs @@ -14,11 +14,11 @@ public sealed class TestRealtimeSession : IRealtimeSession /// Gets or sets the callback to invoke when is called. public Func? UpdateAsyncCallback { get; set; } - /// Gets or sets the callback to invoke when is called. - public Func? InjectClientMessageAsyncCallback { get; set; } + /// Gets or sets the callback to invoke when is called. + public Func? SendClientMessageAsyncCallback { get; set; } /// Gets or sets the callback to invoke when is called. - public Func, CancellationToken, IAsyncEnumerable>? GetStreamingResponseAsyncCallback { get; set; } + public Func>? GetStreamingResponseAsyncCallback { get; set; } /// Gets or sets the callback to invoke when is called. public Func? GetServiceCallback { get; set; } @@ -33,16 +33,16 @@ public Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancel } /// - public Task InjectClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) + public Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) { - return InjectClientMessageAsyncCallback?.Invoke(message, cancellationToken) ?? Task.CompletedTask; + return SendClientMessageAsyncCallback?.Invoke(message, cancellationToken) ?? Task.CompletedTask; } /// public IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default) { - return GetStreamingResponseAsyncCallback?.Invoke(updates, cancellationToken) ?? EmptyAsyncEnumerable(); + return GetStreamingResponseAsyncCallback?.Invoke(cancellationToken) ?? EmptyAsyncEnumerable(); } /// diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs index d03c0621dd3..6ec0ed0fa8c 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs @@ -56,38 +56,24 @@ public async Task UpdateAsync_NullOptions_Throws() } [Fact] - public async Task InjectClientMessageAsync_NullMessage_Throws() + public async Task SendClientMessageAsync_NullMessage_Throws() { using var session = new OpenAIRealtimeSession("key", "model"); - await Assert.ThrowsAsync("message", () => session.InjectClientMessageAsync(null!)); + await Assert.ThrowsAsync("message", () => session.SendClientMessageAsync(null!)); } [Fact] - public async Task InjectClientMessageAsync_CancelledToken_ReturnsSilently() + public async Task SendClientMessageAsync_CancelledToken_ReturnsSilently() { using var session = new OpenAIRealtimeSession("key", "model"); using var cts = new CancellationTokenSource(); cts.Cancel(); // Should not throw when cancellation is requested. - await session.InjectClientMessageAsync(new RealtimeClientMessage(), cts.Token); + await session.SendClientMessageAsync(new RealtimeClientMessage(), cts.Token); Assert.Null(session.Options); } - [Fact] - public async Task GetStreamingResponseAsync_NullUpdates_Throws() - { - using var session = new OpenAIRealtimeSession("key", "model"); - - await Assert.ThrowsAsync("updates", async () => - { - await foreach (var msg in session.GetStreamingResponseAsync(null!)) - { - _ = msg; - } - }); - } - [Fact] public async Task ConnectAsync_CancelledToken_ReturnsFalse() { diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs index a9f6df9b410..9241a02fbbd 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs @@ -51,13 +51,13 @@ public async Task UpdateAsync_DelegatesToInner() } [Fact] - public async Task InjectClientMessageAsync_DelegatesToInner() + public async Task SendClientMessageAsync_DelegatesToInner() { var called = false; var sentMessage = new RealtimeClientMessage { MessageId = "evt_001" }; using var inner = new TestRealtimeSession { - InjectClientMessageAsyncCallback = (msg, _) => + SendClientMessageAsyncCallback = (msg, _) => { Assert.Same(sentMessage, msg); called = true; @@ -66,7 +66,7 @@ public async Task InjectClientMessageAsync_DelegatesToInner() }; using var delegating = new NoOpDelegatingRealtimeSession(inner); - await delegating.InjectClientMessageAsync(sentMessage); + await delegating.SendClientMessageAsync(sentMessage); Assert.True(called); } @@ -76,12 +76,12 @@ public async Task GetStreamingResponseAsync_DelegatesToInner() var expected = new RealtimeServerMessage { Type = RealtimeServerMessageType.Error, MessageId = "evt_002" }; using var inner = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (_, ct) => YieldSingle(expected, ct), + GetStreamingResponseAsyncCallback = (ct) => YieldSingle(expected, ct), }; using var delegating = new NoOpDelegatingRealtimeSession(inner); var messages = new List(); - await foreach (var msg in delegating.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in delegating.GetStreamingResponseAsync()) { messages.Add(msg); } @@ -164,7 +164,7 @@ public async Task UpdateAsync_FlowsCancellationToken() } [Fact] - public async Task InjectClientMessageAsync_FlowsCancellationToken() + public async Task SendClientMessageAsync_FlowsCancellationToken() { CancellationToken capturedToken = default; using var cts = new CancellationTokenSource(); @@ -172,7 +172,7 @@ public async Task InjectClientMessageAsync_FlowsCancellationToken() using var inner = new TestRealtimeSession { - InjectClientMessageAsyncCallback = (msg, ct) => + SendClientMessageAsyncCallback = (msg, ct) => { capturedToken = ct; return Task.CompletedTask; @@ -180,18 +180,10 @@ public async Task InjectClientMessageAsync_FlowsCancellationToken() }; using var delegating = new NoOpDelegatingRealtimeSession(inner); - await delegating.InjectClientMessageAsync(sentMessage, cts.Token); + await delegating.SendClientMessageAsync(sentMessage, cts.Token); Assert.Equal(cts.Token, capturedToken); } - private static async IAsyncEnumerable EmptyUpdates( - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - _ = cancellationToken; - await Task.CompletedTask.ConfigureAwait(false); - yield break; - } - private static async IAsyncEnumerable YieldSingle( RealtimeServerMessage message, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -224,10 +216,10 @@ public DisposableTestRealtimeSession(Action onDispose) public Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancellationToken = default) => Task.CompletedTask; - public Task InjectClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => Task.CompletedTask; + public Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => Task.CompletedTask; public IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, CancellationToken cancellationToken = default) => + CancellationToken cancellationToken = default) => EmptyUpdatesServer(cancellationToken); public object? GetService(Type serviceType, object? serviceKey = null) => null; diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs index fb953217bc3..173cda9ee60 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs @@ -61,21 +61,6 @@ public void MaximumConsecutiveErrorsPerRequest_InvalidValue_Throws() Assert.Equal(0, session.MaximumConsecutiveErrorsPerRequest); } - [Fact] - public async Task GetStreamingResponseAsync_NullUpdates_Throws() - { - using var inner = new TestRealtimeSession(); - using var session = new FunctionInvokingRealtimeSession(inner); - - await Assert.ThrowsAsync("updates", async () => - { - await foreach (var msg in session.GetStreamingResponseAsync(null!)) - { - _ = msg; - } - }); - } - [Fact] public async Task GetStreamingResponseAsync_NoFunctionCalls_PassesThrough() { @@ -87,12 +72,12 @@ public async Task GetStreamingResponseAsync_NoFunctionCalls_PassesThrough() using var inner = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages(serverMessages, ct), + GetStreamingResponseAsyncCallback = (ct) => YieldMessages(serverMessages, ct), }; using var session = new FunctionInvokingRealtimeSession(inner); var received = new List(); - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { received.Add(msg); } @@ -114,11 +99,11 @@ public async Task GetStreamingResponseAsync_FunctionCall_InvokesAndInjectsResult using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [getWeather] }, - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( + GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_001", "get_weather", new Dictionary { ["city"] = "Seattle" }), ], ct), - InjectClientMessageAsyncCallback = (msg, _) => + SendClientMessageAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -128,7 +113,7 @@ public async Task GetStreamingResponseAsync_FunctionCall_InvokesAndInjectsResult using var session = new FunctionInvokingRealtimeSession(inner); var received = new List(); - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { received.Add(msg); } @@ -162,11 +147,11 @@ public async Task GetStreamingResponseAsync_FunctionCall_FromAdditionalTools() var injectedMessages = new List(); using var inner = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( + GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_002", "get_weather", new Dictionary { ["city"] = "London" }), ], ct), - InjectClientMessageAsyncCallback = (msg, _) => + SendClientMessageAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -178,7 +163,7 @@ public async Task GetStreamingResponseAsync_FunctionCall_FromAdditionalTools() AdditionalTools = [getWeather], }; - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { // consume } @@ -208,8 +193,8 @@ public async Task GetStreamingResponseAsync_MaxIterations_StopsInvoking() using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [countFunc] }, - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages(messages, ct), - InjectClientMessageAsyncCallback = (_, _) => Task.CompletedTask, + GetStreamingResponseAsyncCallback = (ct) => YieldMessages(messages, ct), + SendClientMessageAsyncCallback = (_, _) => Task.CompletedTask, }; using var session = new FunctionInvokingRealtimeSession(inner) @@ -218,7 +203,7 @@ public async Task GetStreamingResponseAsync_MaxIterations_StopsInvoking() }; var received = new List(); - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { received.Add(msg); } @@ -242,11 +227,11 @@ public async Task GetStreamingResponseAsync_FunctionInvoker_CustomDelegate() using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [myFunc] }, - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( + GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_custom", "my_func", null), ], ct), - InjectClientMessageAsyncCallback = (_, _) => Task.CompletedTask, + SendClientMessageAsyncCallback = (_, _) => Task.CompletedTask, }; using var session = new FunctionInvokingRealtimeSession(inner) @@ -258,7 +243,7 @@ public async Task GetStreamingResponseAsync_FunctionInvoker_CustomDelegate() }, }; - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { // consume } @@ -272,11 +257,11 @@ public async Task GetStreamingResponseAsync_UnknownFunction_SendsErrorByDefault( var injectedMessages = new List(); using var inner = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( + GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_unknown", "nonexistent_func", null), ], ct), - InjectClientMessageAsyncCallback = (msg, _) => + SendClientMessageAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -285,7 +270,7 @@ public async Task GetStreamingResponseAsync_UnknownFunction_SendsErrorByDefault( using var session = new FunctionInvokingRealtimeSession(inner); - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { // consume } @@ -309,11 +294,11 @@ public async Task GetStreamingResponseAsync_FunctionError_IncludesDetailedErrors using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [failFunc] }, - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( + GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_fail", "fail_func", null), ], ct), - InjectClientMessageAsyncCallback = (msg, _) => + SendClientMessageAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -325,7 +310,7 @@ public async Task GetStreamingResponseAsync_FunctionError_IncludesDetailedErrors IncludeDetailedErrors = true, }; - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { // consume } @@ -348,11 +333,11 @@ public async Task GetStreamingResponseAsync_FunctionError_HidesDetailsWhenNotEna using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [failFunc] }, - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( + GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_fail2", "fail_func", null), ], ct), - InjectClientMessageAsyncCallback = (msg, _) => + SendClientMessageAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -364,7 +349,7 @@ public async Task GetStreamingResponseAsync_FunctionError_HidesDetailsWhenNotEna IncludeDetailedErrors = false, }; - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { // consume } @@ -392,12 +377,12 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_StopsLoop() var injectedMessages = new List(); using var inner = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( + GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_unknown", "nonexistent_func", null), new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseDone, MessageId = "should_not_reach" }, ], ct), - InjectClientMessageAsyncCallback = (msg, _) => + SendClientMessageAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -410,7 +395,7 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_StopsLoop() }; var received = new List(); - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { received.Add(msg); } @@ -428,11 +413,11 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_False_SendsE var injectedMessages = new List(); using var inner = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( + GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_unknown", "nonexistent_func", null), ], ct), - InjectClientMessageAsyncCallback = (msg, _) => + SendClientMessageAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -445,7 +430,7 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_False_SendsE }; var received = new List(); - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { received.Add(msg); } @@ -513,8 +498,8 @@ public async Task GetStreamingResponseAsync_ConcurrentInvocation_InvokesInParall using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [slowFunc] }, - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages([combinedMessage], ct), - InjectClientMessageAsyncCallback = (_, _) => Task.CompletedTask, + GetStreamingResponseAsyncCallback = (ct) => YieldMessages([combinedMessage], ct), + SendClientMessageAsyncCallback = (_, _) => Task.CompletedTask, }; using var session = new FunctionInvokingRealtimeSession(inner) @@ -522,7 +507,7 @@ public async Task GetStreamingResponseAsync_ConcurrentInvocation_InvokesInParall AllowConcurrentInvocation = true, }; - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { // consume } @@ -556,8 +541,8 @@ public async Task GetStreamingResponseAsync_ConsecutiveErrors_ExceedsLimit_Throw using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [failFunc] }, - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages(messages, ct), - InjectClientMessageAsyncCallback = (_, _) => Task.CompletedTask, + GetStreamingResponseAsyncCallback = (ct) => YieldMessages(messages, ct), + SendClientMessageAsyncCallback = (_, _) => Task.CompletedTask, }; using var session = new FunctionInvokingRealtimeSession(inner) @@ -568,7 +553,7 @@ public async Task GetStreamingResponseAsync_ConsecutiveErrors_ExceedsLimit_Throw // Should eventually throw after exceeding the consecutive error limit await Assert.ThrowsAsync(async () => { - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { // consume } @@ -618,12 +603,12 @@ public async Task GetStreamingResponseAsync_NonInvocableTool_TerminatesLoop() using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [declaration] }, - GetStreamingResponseAsyncCallback = (_, ct) => YieldMessages( + GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ CreateFunctionCallOutputItemMessage("call_decl", "my_declaration", null), new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseDone, MessageId = "should_not_reach" }, ], ct), - InjectClientMessageAsyncCallback = (msg, _) => + SendClientMessageAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -633,7 +618,7 @@ public async Task GetStreamingResponseAsync_NonInvocableTool_TerminatesLoop() using var session = new FunctionInvokingRealtimeSession(inner); var received = new List(); - await foreach (var msg in session.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in session.GetStreamingResponseAsync()) { received.Add(msg); } @@ -662,14 +647,6 @@ private static RealtimeServerResponseOutputItemMessage CreateFunctionCallOutputI }; } - private static async IAsyncEnumerable EmptyUpdates( - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - _ = cancellationToken; - await Task.CompletedTask.ConfigureAwait(false); - yield break; - } - private static async IAsyncEnumerable YieldMessages( IList messages, [EnumeratorCancellation] CancellationToken cancellationToken = default) diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs index 5a26dc339d8..9eac51a59ea 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs @@ -90,7 +90,7 @@ public async Task UpdateAsync_LogsInvocationAndCompletion(LogLevel level) [InlineData(LogLevel.Trace)] [InlineData(LogLevel.Debug)] [InlineData(LogLevel.Information)] - public async Task InjectClientMessageAsync_LogsInvocationAndCompletion(LogLevel level) + public async Task SendClientMessageAsync_LogsInvocationAndCompletion(LogLevel level) { var collector = new FakeLogCollector(); @@ -100,7 +100,7 @@ public async Task InjectClientMessageAsync_LogsInvocationAndCompletion(LogLevel using var innerSession = new TestRealtimeSession { - InjectClientMessageAsyncCallback = (message, cancellationToken) => Task.CompletedTask, + SendClientMessageAsyncCallback = (message, cancellationToken) => Task.CompletedTask, }; using var session = innerSession @@ -108,20 +108,20 @@ public async Task InjectClientMessageAsync_LogsInvocationAndCompletion(LogLevel .UseLogging() .Build(services); - await session.InjectClientMessageAsync(new RealtimeClientMessage { MessageId = "test-event-123" }); + await session.SendClientMessageAsync(new RealtimeClientMessage { MessageId = "test-event-123" }); var logs = collector.GetSnapshot(); if (level is LogLevel.Trace) { Assert.Collection(logs, - entry => Assert.Contains("InjectClientMessageAsync invoked:", entry.Message), - entry => Assert.Contains("InjectClientMessageAsync completed.", entry.Message)); + entry => Assert.Contains("SendClientMessageAsync invoked:", entry.Message), + entry => Assert.Contains("SendClientMessageAsync completed.", entry.Message)); } else if (level is LogLevel.Debug) { Assert.Collection(logs, - entry => Assert.Contains("InjectClientMessageAsync invoked.", entry.Message), - entry => Assert.Contains("InjectClientMessageAsync completed.", entry.Message)); + entry => Assert.Contains("SendClientMessageAsync invoked.", entry.Message), + entry => Assert.Contains("SendClientMessageAsync completed.", entry.Message)); } else { @@ -140,7 +140,7 @@ public async Task GetStreamingResponseAsync_LogsMessagesReceived(LogLevel level) using var innerSession = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (messages, cancellationToken) => GetMessagesAsync() + GetStreamingResponseAsyncCallback = (cancellationToken) => GetMessagesAsync() }; static async IAsyncEnumerable GetMessagesAsync() @@ -155,7 +155,7 @@ static async IAsyncEnumerable GetMessagesAsync() .UseLogging(loggerFactory) .Build(); - await foreach (var message in session.GetStreamingResponseAsync(EmptyAsyncEnumerableAsync())) + await foreach (var message in session.GetStreamingResponseAsync()) { // nop } @@ -250,7 +250,7 @@ public async Task GetStreamingResponseAsync_LogsCancellation() using var innerSession = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (messages, cancellationToken) => ThrowCancellationAsync(cancellationToken) + GetStreamingResponseAsyncCallback = (cancellationToken) => ThrowCancellationAsync(cancellationToken) }; static async IAsyncEnumerable ThrowCancellationAsync([EnumeratorCancellation] CancellationToken cancellationToken) @@ -270,7 +270,7 @@ static async IAsyncEnumerable ThrowCancellationAsync([Enu cts.Cancel(); await Assert.ThrowsAsync(async () => { - await foreach (var message in session.GetStreamingResponseAsync(EmptyAsyncEnumerableAsync(), cts.Token)) + await foreach (var message in session.GetStreamingResponseAsync(cts.Token)) { // nop } @@ -290,7 +290,7 @@ public async Task GetStreamingResponseAsync_LogsErrors() using var innerSession = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (messages, cancellationToken) => ThrowErrorAsync() + GetStreamingResponseAsyncCallback = (cancellationToken) => ThrowErrorAsync() }; static async IAsyncEnumerable ThrowErrorAsync() @@ -309,7 +309,7 @@ static async IAsyncEnumerable ThrowErrorAsync() await Assert.ThrowsAsync(async () => { - await foreach (var message in session.GetStreamingResponseAsync(EmptyAsyncEnumerableAsync())) + await foreach (var message in session.GetStreamingResponseAsync()) { // nop } @@ -338,7 +338,7 @@ public void GetService_ReturnsLoggingSessionWhenRequested() } [Fact] - public async Task InjectClientMessageAsync_LogsCancellation() + public async Task SendClientMessageAsync_LogsCancellation() { var collector = new FakeLogCollector(); using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); @@ -347,7 +347,7 @@ public async Task InjectClientMessageAsync_LogsCancellation() using var innerSession = new TestRealtimeSession { - InjectClientMessageAsyncCallback = (message, cancellationToken) => + SendClientMessageAsyncCallback = (message, cancellationToken) => { throw new OperationCanceledException(cancellationToken); }, @@ -360,23 +360,23 @@ public async Task InjectClientMessageAsync_LogsCancellation() cts.Cancel(); await Assert.ThrowsAsync(() => - session.InjectClientMessageAsync(new RealtimeClientMessage { MessageId = "evt_cancel" }, cts.Token)); + session.SendClientMessageAsync(new RealtimeClientMessage { MessageId = "evt_cancel" }, cts.Token)); var logs = collector.GetSnapshot(); Assert.Collection(logs, - entry => Assert.Contains("InjectClientMessageAsync invoked.", entry.Message), - entry => Assert.Contains("InjectClientMessageAsync canceled.", entry.Message)); + entry => Assert.Contains("SendClientMessageAsync invoked.", entry.Message), + entry => Assert.Contains("SendClientMessageAsync canceled.", entry.Message)); } [Fact] - public async Task InjectClientMessageAsync_LogsErrors() + public async Task SendClientMessageAsync_LogsErrors() { var collector = new FakeLogCollector(); using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); using var innerSession = new TestRealtimeSession { - InjectClientMessageAsyncCallback = (message, cancellationToken) => + SendClientMessageAsyncCallback = (message, cancellationToken) => { throw new InvalidOperationException("Inject error"); }, @@ -388,70 +388,12 @@ public async Task InjectClientMessageAsync_LogsErrors() .Build(); await Assert.ThrowsAsync(() => - session.InjectClientMessageAsync(new RealtimeClientMessage())); + session.SendClientMessageAsync(new RealtimeClientMessage())); var logs = collector.GetSnapshot(); Assert.Collection(logs, - entry => Assert.Contains("InjectClientMessageAsync invoked.", entry.Message), - entry => Assert.True(entry.Message.Contains("InjectClientMessageAsync failed.") && entry.Level == LogLevel.Error)); - } - - [Theory] - [InlineData(LogLevel.Trace)] - [InlineData(LogLevel.Debug)] - [InlineData(LogLevel.Information)] - public async Task GetStreamingResponseAsync_LogsClientMessages(LogLevel level) - { - var collector = new FakeLogCollector(); - using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(level)); - - using var innerSession = new TestRealtimeSession - { - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => ConsumeAndYield(updates, cancellationToken) - }; - - static async IAsyncEnumerable ConsumeAndYield( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) - { - await foreach (var update in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // consume - } - - yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseDone }; - } - - using var session = innerSession - .AsBuilder() - .UseLogging(loggerFactory) - .Build(); - - var clientMessages = GetClientMessages(); - await foreach (var message in session.GetStreamingResponseAsync(clientMessages)) - { - // consume - } - - var logs = collector.GetSnapshot(); - if (level is LogLevel.Trace) - { - // Should log: invoked, client message (sensitive), server message (sensitive), completed - Assert.True(logs.Count >= 3); - Assert.Contains(logs, entry => entry.Message.Contains("GetStreamingResponseAsync invoked.")); - Assert.Contains(logs, entry => entry.Message.Contains("sending client message:")); - Assert.Contains(logs, entry => entry.Message.Contains("GetStreamingResponseAsync completed.")); - } - else if (level is LogLevel.Debug) - { - Assert.True(logs.Count >= 3); - Assert.Contains(logs, entry => entry.Message.Contains("GetStreamingResponseAsync invoked.")); - Assert.Contains(logs, entry => entry.Message.Contains("sending client message.")); - Assert.Contains(logs, entry => entry.Message.Contains("GetStreamingResponseAsync completed.")); - } - else - { - Assert.Empty(logs); - } + entry => Assert.Contains("SendClientMessageAsync invoked.", entry.Message), + entry => Assert.True(entry.Message.Contains("SendClientMessageAsync failed.") && entry.Level == LogLevel.Error)); } [Fact] @@ -500,17 +442,4 @@ public void UseLogging_ConfigureCallback_IsInvoked() Assert.True(configured); } - private static async IAsyncEnumerable EmptyAsyncEnumerableAsync() - { - await Task.CompletedTask.ConfigureAwait(false); - yield break; - } - - private static async IAsyncEnumerable GetClientMessages( - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - _ = cancellationToken; - await Task.CompletedTask.ConfigureAwait(false); - yield return new RealtimeClientMessage { MessageId = "client_evt_1" }; - } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs index 79552329be4..5494b202a3a 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs @@ -44,19 +44,13 @@ public async Task ExpectedInformationLogged_GetStreamingResponseAsync(bool enabl GetServiceCallback = (serviceType, serviceKey) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("testprovider", new Uri("http://localhost:12345/realtime"), "gpt-4-realtime") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => CallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackAsync(cancellationToken), }; - static async IAsyncEnumerable CallbackAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) + static async IAsyncEnumerable CallbackAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); - - // Consume the incoming updates - await foreach (var update in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Just consume the update - } + _ = cancellationToken; yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseCreated, MessageId = "evt_001" }; yield return new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputTextDelta) { OutputIndex = 0, Text = "Hello" }; @@ -91,8 +85,12 @@ static async IAsyncEnumerable CallbackAsync( }) .Build(); - var clientMessages = GetClientMessagesAsync(); - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume responses } @@ -275,13 +273,13 @@ public async Task GetStreamingResponseAsync_TracesError() using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => ThrowingCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => ThrowingCallbackAsync(cancellationToken), }; - static async IAsyncEnumerable ThrowingCallbackAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) + static async IAsyncEnumerable ThrowingCallbackAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); + _ = cancellationToken; yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseCreated }; throw new InvalidOperationException("Streaming error"); } @@ -291,10 +289,9 @@ static async IAsyncEnumerable ThrowingCallbackAsync( .UseOpenTelemetry(sourceName: sourceName) .Build(); - var clientMessages = GetClientMessagesAsync(); await Assert.ThrowsAsync(async () => { - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume responses } @@ -319,18 +316,13 @@ public async Task GetStreamingResponseAsync_TracesErrorFromResponse() using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => ErrorResponseCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => ErrorResponseCallbackAsync(cancellationToken), }; - static async IAsyncEnumerable ErrorResponseCallbackAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) + static async IAsyncEnumerable ErrorResponseCallbackAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); - - await foreach (var _ in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Consume - } + _ = cancellationToken; yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone) { @@ -345,8 +337,12 @@ static async IAsyncEnumerable ErrorResponseCallbackAsync( .UseOpenTelemetry(sourceName: sourceName) .Build(); - var clientMessages = GetClientMessagesAsync(); - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume responses } @@ -374,18 +370,13 @@ public async Task DefaultVoiceSpeed_NotLogged() Model = "test-model", VoiceSpeed = 1.0, // Default value should not be logged }, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => EmptyCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => EmptyCallbackAsync(cancellationToken), }; - static async IAsyncEnumerable EmptyCallbackAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) + static async IAsyncEnumerable EmptyCallbackAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); - - await foreach (var _ in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Consume - } + _ = cancellationToken; yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); } @@ -395,8 +386,12 @@ static async IAsyncEnumerable EmptyCallbackAsync( .UseOpenTelemetry(sourceName: sourceName) .Build(); - var clientMessages = GetClientMessagesAsync(); - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -417,18 +412,15 @@ public async Task NoListeners_NoActivityCreated() using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => EmptyCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => EmptyCallbackAsync(cancellationToken), }; - static async IAsyncEnumerable EmptyCallbackAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) +#pragma warning disable S4144 // Methods should not have identical implementations + static async IAsyncEnumerable EmptyCallbackAsync([EnumeratorCancellation] CancellationToken cancellationToken) +#pragma warning restore S4144 { await Task.Yield(); - - await foreach (var _ in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Consume - } + _ = cancellationToken; yield break; } @@ -440,9 +432,8 @@ static async IAsyncEnumerable EmptyCallbackAsync( .Build(); // This should work without errors even without listeners - var clientMessages = GetClientMessagesAsync(); var count = 0; - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var response in session.GetStreamingResponseAsync()) { count++; } @@ -469,21 +460,6 @@ public async Task UpdateAsync_InvalidArgs_Throws() await Assert.ThrowsAsync("options", () => session.UpdateAsync(null!)); } - [Fact] - public async Task GetStreamingResponseAsync_InvalidArgs_Throws() - { - using var innerSession = new TestRealtimeSession(); - using var session = new OpenTelemetryRealtimeSession(innerSession); - - await Assert.ThrowsAsync("updates", async () => - { - await foreach (var _ in session.GetStreamingResponseAsync(null!)) - { - // Should not reach here - } - }); - } - [Fact] public void GetService_ReturnsActivitySource() { @@ -525,18 +501,13 @@ public async Task TranscriptionSessionKind_Logged() Model = "whisper-1", SessionKind = RealtimeSessionKind.Transcription, }, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => TranscriptionCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => TranscriptionCallbackAsync(cancellationToken), }; - static async IAsyncEnumerable TranscriptionCallbackAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) + static async IAsyncEnumerable TranscriptionCallbackAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); - - await foreach (var _ in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Consume - } + _ = cancellationToken; yield return new RealtimeServerInputAudioTranscriptionMessage(RealtimeServerMessageType.InputAudioTranscriptionCompleted) { @@ -550,8 +521,12 @@ static async IAsyncEnumerable TranscriptionCallbackAsync( .UseOpenTelemetry(sourceName: sourceName) .Build(); - var clientMessages = GetClientMessagesAsync(); - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -589,7 +564,7 @@ public async Task ToolChoiceMode_Logged(string modeKey, string expectedValue) ToolMode = mode, Tools = [AIFunctionFactory.Create((string query) => query, "Search")], }, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -597,8 +572,12 @@ public async Task ToolChoiceMode_Logged(string modeKey, string expectedValue) .UseOpenTelemetry(sourceName: sourceName) .Build(); - var clientMessages = GetClientMessagesAsync(); - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -624,7 +603,7 @@ public async Task AIFunction_ForcedTool_Logged() Model = "test-model", ToolMode = ChatToolMode.RequireSpecific("SpecificSearch"), }, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -632,8 +611,12 @@ public async Task AIFunction_ForcedTool_Logged() .UseOpenTelemetry(sourceName: sourceName) .Build(); - var clientMessages = GetClientMessagesAsync(); - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -660,7 +643,7 @@ public async Task RequireAny_ToolMode_Logged() Model = "test-model", ToolMode = ChatToolMode.RequireAny, }, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -668,8 +651,12 @@ public async Task RequireAny_ToolMode_Logged() .UseOpenTelemetry(sourceName: sourceName) .Build(); - var clientMessages = GetClientMessagesAsync(); - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -695,7 +682,7 @@ public async Task NoToolChoice_NotLogged() { Model = "test-model", }, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -703,8 +690,12 @@ public async Task NoToolChoice_NotLogged() .UseOpenTelemetry(sourceName: sourceName) .Build(); - var clientMessages = GetClientMessagesAsync(); - await foreach (var response in session.GetStreamingResponseAsync(clientMessages)) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -728,7 +719,7 @@ public async Task ToolCallContentInClientMessages_LoggedAsInputMessages() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -736,7 +727,12 @@ public async Task ToolCallContentInClientMessages_LoggedAsInputMessages() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesWithToolResultAsync())) + await foreach (var msg in GetClientMessagesWithToolResultAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -766,7 +762,7 @@ public async Task ToolCallContentInServerMessages_LoggedAsOutputMessages() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => CallbackWithToolCallAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithToolCallAsync(cancellationToken), }; using var session = innerSession @@ -774,7 +770,12 @@ public async Task ToolCallContentInServerMessages_LoggedAsOutputMessages() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesAsync())) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -804,7 +805,7 @@ public async Task ToolContentNotLoggedWithoutSensitiveData() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => CallbackWithToolCallAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithToolCallAsync(cancellationToken), }; using var session = innerSession @@ -812,7 +813,12 @@ public async Task ToolContentNotLoggedWithoutSensitiveData() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = false) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesWithToolResultAsync())) + await foreach (var msg in GetClientMessagesWithToolResultAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -822,15 +828,12 @@ public async Task ToolContentNotLoggedWithoutSensitiveData() Assert.Null(activity.GetTagItem("gen_ai.output.messages")); } - private static async IAsyncEnumerable SimpleCallbackAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) +#pragma warning disable S4144 // Methods should not have identical implementations + private static async IAsyncEnumerable SimpleCallbackAsync([EnumeratorCancellation] CancellationToken cancellationToken) +#pragma warning restore S4144 { await Task.Yield(); - - await foreach (var _ in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Consume - } + _ = cancellationToken; yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); } @@ -855,16 +858,10 @@ private static async IAsyncEnumerable GetClientMessagesWi yield return new RealtimeClientResponseCreateMessage(); } - private static async IAsyncEnumerable CallbackWithToolCallAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) + private static async IAsyncEnumerable CallbackWithToolCallAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); - - // Consume incoming messages - await foreach (var _ in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Consume - } + _ = cancellationToken; // Yield a function call item from the server using RealtimeServerResponseOutputItemMessage var contentItem = new RealtimeContentItem( @@ -893,7 +890,7 @@ public async Task AudioBufferAppendMessage_LoggedAsInputMessage() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -901,7 +898,12 @@ public async Task AudioBufferAppendMessage_LoggedAsInputMessage() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesAsync())) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -931,7 +933,7 @@ public async Task AudioBufferCommitMessage_LoggedAsInputMessage() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -939,7 +941,12 @@ public async Task AudioBufferCommitMessage_LoggedAsInputMessage() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesAsync())) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -969,7 +976,7 @@ public async Task ResponseCreateMessageWithInstructions_LoggedAsInputMessage() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -977,7 +984,12 @@ public async Task ResponseCreateMessageWithInstructions_LoggedAsInputMessage() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesWithInstructionsAsync())) + await foreach (var msg in GetClientMessagesWithInstructionsAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -1006,7 +1018,7 @@ public async Task ResponseCreateMessageWithItems_LoggedAsInputMessage() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -1014,7 +1026,12 @@ public async Task ResponseCreateMessageWithItems_LoggedAsInputMessage() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesWithItemsAsync())) + await foreach (var msg in GetClientMessagesWithItemsAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -1043,7 +1060,7 @@ public async Task OutputTextAudioMessage_LoggedAsOutputMessage() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => CallbackWithTextOutputAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithTextOutputAsync(cancellationToken), }; using var session = innerSession @@ -1051,7 +1068,12 @@ public async Task OutputTextAudioMessage_LoggedAsOutputMessage() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesAsync())) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -1079,7 +1101,7 @@ public async Task InputAudioTranscriptionMessage_LoggedAsOutputMessage() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => CallbackWithTranscriptionAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithTranscriptionAsync(cancellationToken), }; using var session = innerSession @@ -1087,7 +1109,12 @@ public async Task InputAudioTranscriptionMessage_LoggedAsOutputMessage() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesAsync())) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -1115,7 +1142,7 @@ public async Task ServerErrorMessage_LoggedAsOutputMessage() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => CallbackWithServerErrorAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithServerErrorAsync(cancellationToken), }; using var session = innerSession @@ -1123,7 +1150,12 @@ public async Task ServerErrorMessage_LoggedAsOutputMessage() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesAsync())) + await foreach (var msg in GetClientMessagesAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -1151,7 +1183,7 @@ public async Task ConversationItemCreateWithTextContent_LoggedAsInputMessage() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -1159,7 +1191,12 @@ public async Task ConversationItemCreateWithTextContent_LoggedAsInputMessage() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesWithTextContentAsync())) + await foreach (var msg in GetClientMessagesWithTextContentAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -1187,7 +1224,7 @@ public async Task DataContentInClientMessage_LoggedWithModality() Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("test-provider") : null, - GetStreamingResponseAsyncCallback = (updates, cancellationToken) => SimpleCallbackAsync(updates, cancellationToken), + GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; using var session = innerSession @@ -1195,7 +1232,12 @@ public async Task DataContentInClientMessage_LoggedWithModality() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); - await foreach (var response in session.GetStreamingResponseAsync(GetClientMessagesWithImageContentAsync())) + await foreach (var msg in GetClientMessagesWithImageContentAsync()) + { + await session.SendClientMessageAsync(msg); + } + + await foreach (var response in session.GetStreamingResponseAsync()) { // Consume } @@ -1247,15 +1289,10 @@ private static async IAsyncEnumerable GetClientMessagesWi yield return new RealtimeClientResponseCreateMessage(); } - private static async IAsyncEnumerable CallbackWithTextOutputAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) + private static async IAsyncEnumerable CallbackWithTextOutputAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); - - await foreach (var _ in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Consume - } + _ = cancellationToken; yield return new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputTextDone) { @@ -1264,15 +1301,10 @@ private static async IAsyncEnumerable CallbackWithTextOut yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); } - private static async IAsyncEnumerable CallbackWithTranscriptionAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) + private static async IAsyncEnumerable CallbackWithTranscriptionAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); - - await foreach (var _ in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Consume - } + _ = cancellationToken; yield return new RealtimeServerInputAudioTranscriptionMessage(RealtimeServerMessageType.InputAudioTranscriptionCompleted) { @@ -1281,15 +1313,10 @@ private static async IAsyncEnumerable CallbackWithTranscr yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); } - private static async IAsyncEnumerable CallbackWithServerErrorAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken) + private static async IAsyncEnumerable CallbackWithServerErrorAsync([EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); - - await foreach (var _ in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) - { - // Consume - } + _ = cancellationToken; yield return new RealtimeServerErrorMessage { diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs index 25b10374d75..b39e698c57f 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs @@ -64,7 +64,7 @@ public void Use_StreamingDelegate_NullFunc_Throws() Assert.Throws( "getStreamingResponseFunc", - () => builder.Use((Func, IRealtimeSession, CancellationToken, IAsyncEnumerable>)null!)); + () => builder.Use((Func>)null!)); } [Fact] @@ -142,18 +142,18 @@ public async Task Use_WithStreamingDelegate_InterceptsStreaming() var intercepted = false; using var inner = new TestRealtimeSession { - GetStreamingResponseAsyncCallback = (_, ct) => YieldSingle(new RealtimeServerMessage { MessageId = "inner" }, ct), + GetStreamingResponseAsyncCallback = (ct) => YieldSingle(new RealtimeServerMessage { MessageId = "inner" }, ct), }; var builder = new RealtimeSessionBuilder(inner); - builder.Use((updates, innerSession, ct) => + builder.Use((innerSession, ct) => { intercepted = true; - return innerSession.GetStreamingResponseAsync(updates, ct); + return innerSession.GetStreamingResponseAsync(ct); }); using var pipeline = builder.Build(); - await foreach (var msg in pipeline.GetStreamingResponseAsync(EmptyUpdates())) + await foreach (var msg in pipeline.GetStreamingResponseAsync()) { Assert.Equal("inner", msg.MessageId); } @@ -177,14 +177,6 @@ public void AsBuilder_ReturnsBuilder() Assert.Same(inner, builder.Build()); } - private static async IAsyncEnumerable EmptyUpdates( - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - _ = cancellationToken; - await Task.CompletedTask.ConfigureAwait(false); - yield break; - } - private static async IAsyncEnumerable YieldSingle( RealtimeServerMessage message, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -209,11 +201,10 @@ public OrderTrackingSession(IRealtimeSession inner, string name, List ca public IRealtimeSession GetInner() => InnerSession; public override async IAsyncEnumerable GetStreamingResponseAsync( - IAsyncEnumerable updates, [EnumeratorCancellation] CancellationToken cancellationToken = default) { _callOrder.Add(Name); - await foreach (var msg in base.GetStreamingResponseAsync(updates, cancellationToken).ConfigureAwait(false)) + await foreach (var msg in base.GetStreamingResponseAsync(cancellationToken).ConfigureAwait(false)) { yield return msg; } From 02fd006ca38b9f3817f80f0e6d51ea565287c46b Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Mon, 23 Feb 2026 12:23:12 -0800 Subject: [PATCH 37/92] Make RealtimeSessionOptions and related types immutable with init-only properties and IReadOnlyList --- .../Realtime/RealtimeAudioFormat.cs | 8 +- .../Realtime/RealtimeSessionOptions.cs | 60 +++--- .../SemanticVoiceActivityDetection.cs | 4 +- .../Realtime/ServerVoiceActivityDetection.cs | 16 +- .../Realtime/VoiceActivityDetection.cs | 8 +- .../OpenAIRealtimeSession.cs | 185 ++++++++---------- .../FunctionInvokingRealtimeSession.cs | 4 +- .../Realtime/RealtimeSessionOptionsTests.cs | 40 ++-- 8 files changed, 153 insertions(+), 172 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs index 93c97d08fd2..3d8962c6780 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs @@ -22,16 +22,16 @@ public RealtimeAudioFormat(string mediaType, int sampleRate) } /// - /// Gets or sets the media type of the audio (e.g., "audio/pcm", "audio/pcmu", "audio/pcma"). + /// Gets the media type of the audio (e.g., "audio/pcm", "audio/pcmu", "audio/pcma"). /// - public string MediaType { get; set; } + public string MediaType { get; init; } /// - /// Gets or sets the sample rate of the audio in Hertz. + /// Gets the sample rate of the audio in Hertz. /// /// /// When constructed via , this property is always set. /// The nullable type allows deserialized instances to omit the sample rate when the server does not provide one. /// - public int? SampleRate { get; set; } + public int? SampleRate { get; init; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs index 6180a12ab87..03a02b73187 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs @@ -14,84 +14,84 @@ namespace Microsoft.Extensions.AI; public class RealtimeSessionOptions { /// - /// Gets or sets the session kind. + /// Gets the session kind. /// /// /// If set to , most of the sessions properties will not apply to the session. Only InputAudioFormat, NoiseReductionOptions, TranscriptionOptions, and VoiceActivityDetection will be used. /// - public RealtimeSessionKind SessionKind { get; set; } = RealtimeSessionKind.Realtime; + public RealtimeSessionKind SessionKind { get; init; } = RealtimeSessionKind.Realtime; /// - /// Gets or sets the model name to use for the session. + /// Gets the model name to use for the session. /// - public string? Model { get; set; } + public string? Model { get; init; } /// - /// Gets or sets the input audio format for the session. + /// Gets the input audio format for the session. /// - public RealtimeAudioFormat? InputAudioFormat { get; set; } + public RealtimeAudioFormat? InputAudioFormat { get; init; } /// - /// Gets or sets the noise reduction options for the session. + /// Gets the noise reduction options for the session. /// - public NoiseReductionOptions? NoiseReductionOptions { get; set; } + public NoiseReductionOptions? NoiseReductionOptions { get; init; } /// - /// Gets or sets the transcription options for the session. + /// Gets the transcription options for the session. /// - public TranscriptionOptions? TranscriptionOptions { get; set; } + public TranscriptionOptions? TranscriptionOptions { get; init; } /// - /// Gets or sets the voice activity detection options for the session. + /// Gets the voice activity detection options for the session. /// - public VoiceActivityDetection? VoiceActivityDetection { get; set; } + public VoiceActivityDetection? VoiceActivityDetection { get; init; } /// - /// Gets or sets the output audio format for the session. + /// Gets the output audio format for the session. /// - public RealtimeAudioFormat? OutputAudioFormat { get; set; } + public RealtimeAudioFormat? OutputAudioFormat { get; init; } /// - /// Gets or sets the output voice speed for the session. + /// Gets the output voice speed for the session. /// /// /// The default value is 1.0, which represents normal speed. /// - public double VoiceSpeed { get; set; } = 1.0; + public double VoiceSpeed { get; init; } = 1.0; /// - /// Gets or sets the output voice for the session. + /// Gets the output voice for the session. /// - public string? Voice { get; set; } + public string? Voice { get; init; } /// - /// Gets or sets the default system instructions for the session. + /// Gets the default system instructions for the session. /// - public string? Instructions { get; set; } + public string? Instructions { get; init; } /// - /// Gets or sets the maximum number of response tokens for the session. + /// Gets the maximum number of response tokens for the session. /// - public int? MaxOutputTokens { get; set; } + public int? MaxOutputTokens { get; init; } /// - /// Gets or sets the output modalities for the response. like "text", "audio". + /// Gets the output modalities for the response. like "text", "audio". /// If null, then default conversation modalities will be used. /// - public IList? OutputModalities { get; set; } + public IReadOnlyList? OutputModalities { get; init; } /// - /// Gets or sets the tool choice mode for the session. + /// Gets the tool choice mode for the session. /// - public ChatToolMode? ToolMode { get; set; } + public ChatToolMode? ToolMode { get; init; } /// - /// Gets or sets the AI tools available for generating the response. + /// Gets the AI tools available for generating the response. /// - public IList? Tools { get; set; } + public IReadOnlyList? Tools { get; init; } /// - /// Gets or sets a callback responsible for creating the raw representation of the session options from an underlying implementation. + /// Gets a callback responsible for creating the raw representation of the session options from an underlying implementation. /// /// /// The underlying implementation might have its own representation of options. @@ -108,5 +108,5 @@ public class RealtimeSessionOptions /// properties on . /// [JsonIgnore] - public Func? RawRepresentationFactory { get; set; } + public Func? RawRepresentationFactory { get; init; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs index 12f995d5b42..c4c94f3b7f5 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs @@ -13,7 +13,7 @@ namespace Microsoft.Extensions.AI; public class SemanticVoiceActivityDetection : VoiceActivityDetection { /// - /// Gets or sets the eagerness level for semantic voice activity detection. + /// Gets the eagerness level for semantic voice activity detection. /// - public SemanticEagerness Eagerness { get; set; } = SemanticEagerness.Auto; + public SemanticEagerness Eagerness { get; init; } = SemanticEagerness.Auto; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs index bdea1f39cbb..7b0946337ef 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs @@ -13,25 +13,25 @@ namespace Microsoft.Extensions.AI; public class ServerVoiceActivityDetection : VoiceActivityDetection { /// - /// Gets or sets the idle timeout in milliseconds to detect the end of speech. + /// Gets the idle timeout in milliseconds to detect the end of speech. /// - public int IdleTimeoutInMilliseconds { get; set; } + public int IdleTimeoutInMilliseconds { get; init; } /// - /// Gets or sets the prefix padding in milliseconds to include before detected speech. + /// Gets the prefix padding in milliseconds to include before detected speech. /// - public int PrefixPaddingInMilliseconds { get; set; } = 300; + public int PrefixPaddingInMilliseconds { get; init; } = 300; /// - /// Gets or sets the silence duration in milliseconds to consider as a pause. + /// Gets the silence duration in milliseconds to consider as a pause. /// - public int SilenceDurationInMilliseconds { get; set; } = 500; + public int SilenceDurationInMilliseconds { get; init; } = 500; /// - /// Gets or sets the threshold for voice activity detection. + /// Gets the threshold for voice activity detection. /// /// /// A value between 0.0 and 1.0, where higher values make the detection more sensitive. /// - public double Threshold { get; set; } = 0.5; + public double Threshold { get; init; } = 0.5; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs index d244cf24307..aa6c58c00f7 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs @@ -13,12 +13,12 @@ namespace Microsoft.Extensions.AI; public class VoiceActivityDetection { /// - /// Gets or sets a value indicating whether to create a response when voice activity is detected. + /// Gets a value indicating whether to create a response when voice activity is detected. /// - public bool CreateResponse { get; set; } + public bool CreateResponse { get; init; } /// - /// Gets or sets a value indicating whether to interrupt the response when voice activity is detected. + /// Gets a value indicating whether to interrupt the response when voice activity is detected. /// - public bool InterruptResponse { get; set; } + public bool InterruptResponse { get; init; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index e3662821608..52571623ead 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -1315,17 +1315,7 @@ [new McpServerToolApprovalRequestContent(approvalId, toolCall) { RawRepresentati { if (root.TryGetProperty("session", out var sessionElement)) { - var newOptions = DeserializeSessionOptions(sessionElement); - - // Preserve client-side properties that the server cannot round-trip - // as typed objects (tools are returned as JSON schemas, not AITool instances). - if (Options is not null) - { - newOptions.Tools = Options.Tools; - newOptions.ToolMode = Options.ToolMode; - } - - Options = newOptions; + Options = DeserializeSessionOptions(sessionElement, Options); } return new RealtimeServerMessage @@ -1335,37 +1325,35 @@ [new McpServerToolApprovalRequestContent(approvalId, toolCall) { RawRepresentati }; } - private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement session) + private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement session, RealtimeSessionOptions? previousOptions) { - var options = new RealtimeSessionOptions(); - - if (session.TryGetProperty("type", out var typeElement)) + RealtimeSessionKind sessionKind = RealtimeSessionKind.Realtime; + if (session.TryGetProperty("type", out var typeElement) && typeElement.GetString() == "transcription") { - options.SessionKind = typeElement.GetString() == "transcription" - ? RealtimeSessionKind.Transcription - : RealtimeSessionKind.Realtime; + sessionKind = RealtimeSessionKind.Transcription; } - if (session.TryGetProperty("model", out var modelElement)) - { - options.Model = modelElement.GetString(); - } + string? model = session.TryGetProperty("model", out var modelElement) ? modelElement.GetString() : null; - if (session.TryGetProperty("instructions", out var instructionsElement) && - instructionsElement.ValueKind == JsonValueKind.String) - { - options.Instructions = instructionsElement.GetString(); - } + string? instructions = session.TryGetProperty("instructions", out var instructionsElement) && instructionsElement.ValueKind == JsonValueKind.String + ? instructionsElement.GetString() + : null; - if (session.TryGetProperty("max_output_tokens", out var maxTokensElement)) - { - options.MaxOutputTokens = ParseMaxOutputTokens(maxTokensElement); - } + int? maxOutputTokens = session.TryGetProperty("max_output_tokens", out var maxTokensElement) + ? ParseMaxOutputTokens(maxTokensElement) + : null; - if (session.TryGetProperty("output_modalities", out var modalitiesElement)) - { - options.OutputModalities = ParseOutputModalities(modalitiesElement); - } + IReadOnlyList? outputModalities = session.TryGetProperty("output_modalities", out var modalitiesElement) + ? ParseOutputModalities(modalitiesElement) + : null; + + RealtimeAudioFormat? inputAudioFormat = null; + NoiseReductionOptions? noiseReductionOptions = null; + TranscriptionOptions? transcriptionOptions = null; + VoiceActivityDetection? voiceActivityDetection = null; + RealtimeAudioFormat? outputAudioFormat = null; + double voiceSpeed = 1.0; + string? voice = null; // Audio configuration. if (session.TryGetProperty("audio", out var audioElement) && @@ -1377,14 +1365,14 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess { if (inputElement.TryGetProperty("format", out var inputFormatElement)) { - options.InputAudioFormat = ParseAudioFormat(inputFormatElement); + inputAudioFormat = ParseAudioFormat(inputFormatElement); } if (inputElement.TryGetProperty("noise_reduction", out var noiseElement) && noiseElement.ValueKind == JsonValueKind.Object && noiseElement.TryGetProperty("type", out var noiseTypeElement)) { - options.NoiseReductionOptions = noiseTypeElement.GetString() switch + noiseReductionOptions = noiseTypeElement.GetString() switch { "near_field" => NoiseReductionOptions.NearField, "far_field" => NoiseReductionOptions.FarField, @@ -1396,12 +1384,12 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess transcriptionElement.ValueKind == JsonValueKind.Object) { string? language = transcriptionElement.TryGetProperty("language", out var langElement) ? langElement.GetString() : null; - string? model = transcriptionElement.TryGetProperty("model", out var modelEl) ? modelEl.GetString() : null; + string? transcriptionModel = transcriptionElement.TryGetProperty("model", out var modelEl) ? modelEl.GetString() : null; string? prompt = transcriptionElement.TryGetProperty("prompt", out var promptElement) ? promptElement.GetString() : null; - if (language is not null && model is not null) + if (language is not null && transcriptionModel is not null) { - options.TranscriptionOptions = new TranscriptionOptions { SpeechLanguage = language, ModelId = model, Prompt = prompt }; + transcriptionOptions = new TranscriptionOptions { SpeechLanguage = language, ModelId = transcriptionModel, Prompt = prompt }; } } @@ -1410,63 +1398,7 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess turnDetectionElement.ValueKind == JsonValueKind.Object && turnDetectionElement.TryGetProperty("type", out var vadTypeElement)) { - string? vadType = vadTypeElement.GetString(); - if (vadType == "server_vad") - { - var serverVad = new ServerVoiceActivityDetection(); - if (turnDetectionElement.TryGetProperty("create_response", out var crElement)) - { - serverVad.CreateResponse = crElement.GetBoolean(); - } - - if (turnDetectionElement.TryGetProperty("interrupt_response", out var irElement)) - { - serverVad.InterruptResponse = irElement.GetBoolean(); - } - - if (turnDetectionElement.TryGetProperty("idle_timeout_ms", out var itElement) && itElement.ValueKind == JsonValueKind.Number) - { - serverVad.IdleTimeoutInMilliseconds = itElement.GetInt32(); - } - - if (turnDetectionElement.TryGetProperty("prefix_padding_ms", out var ppElement) && ppElement.ValueKind == JsonValueKind.Number) - { - serverVad.PrefixPaddingInMilliseconds = ppElement.GetInt32(); - } - - if (turnDetectionElement.TryGetProperty("silence_duration_ms", out var sdElement) && sdElement.ValueKind == JsonValueKind.Number) - { - serverVad.SilenceDurationInMilliseconds = sdElement.GetInt32(); - } - - if (turnDetectionElement.TryGetProperty("threshold", out var thElement) && thElement.ValueKind == JsonValueKind.Number) - { - serverVad.Threshold = thElement.GetDouble(); - } - - options.VoiceActivityDetection = serverVad; - } - else if (vadType == "semantic_vad") - { - var semanticVad = new SemanticVoiceActivityDetection(); - if (turnDetectionElement.TryGetProperty("create_response", out var crElement)) - { - semanticVad.CreateResponse = crElement.GetBoolean(); - } - - if (turnDetectionElement.TryGetProperty("interrupt_response", out var irElement)) - { - semanticVad.InterruptResponse = irElement.GetBoolean(); - } - - if (turnDetectionElement.TryGetProperty("eagerness", out var eagernessElement) && - eagernessElement.GetString() is string eagerness) - { - semanticVad.Eagerness = new SemanticEagerness(eagerness); - } - - options.VoiceActivityDetection = semanticVad; - } + voiceActivityDetection = ParseVoiceActivityDetection(vadTypeElement.GetString(), turnDetectionElement); } } @@ -1476,30 +1408,79 @@ private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement sess { if (outputElement.TryGetProperty("format", out var outputFormatElement)) { - options.OutputAudioFormat = ParseAudioFormat(outputFormatElement); + outputAudioFormat = ParseAudioFormat(outputFormatElement); } if (outputElement.TryGetProperty("speed", out var speedElement) && speedElement.ValueKind == JsonValueKind.Number) { - options.VoiceSpeed = speedElement.GetDouble(); + voiceSpeed = speedElement.GetDouble(); } if (outputElement.TryGetProperty("voice", out var voiceElement)) { if (voiceElement.ValueKind == JsonValueKind.String) { - options.Voice = voiceElement.GetString(); + voice = voiceElement.GetString(); } else if (voiceElement.ValueKind == JsonValueKind.Object && voiceElement.TryGetProperty("id", out var voiceIdElement)) { - options.Voice = voiceIdElement.GetString(); + voice = voiceIdElement.GetString(); } } } } - return options; + return new RealtimeSessionOptions + { + SessionKind = sessionKind, + Model = model, + Instructions = instructions, + MaxOutputTokens = maxOutputTokens, + OutputModalities = outputModalities, + InputAudioFormat = inputAudioFormat, + NoiseReductionOptions = noiseReductionOptions, + TranscriptionOptions = transcriptionOptions, + VoiceActivityDetection = voiceActivityDetection, + OutputAudioFormat = outputAudioFormat, + VoiceSpeed = voiceSpeed, + Voice = voice, + + // Preserve client-side properties that the server cannot round-trip + // as typed objects (tools are returned as JSON schemas, not AITool instances). + Tools = previousOptions?.Tools, + ToolMode = previousOptions?.ToolMode, + }; + } + + private static VoiceActivityDetection? ParseVoiceActivityDetection(string? vadType, JsonElement turnDetectionElement) + { + if (vadType == "server_vad") + { + return new ServerVoiceActivityDetection + { + CreateResponse = turnDetectionElement.TryGetProperty("create_response", out var crElement) && crElement.GetBoolean(), + InterruptResponse = turnDetectionElement.TryGetProperty("interrupt_response", out var irElement) && irElement.GetBoolean(), + IdleTimeoutInMilliseconds = turnDetectionElement.TryGetProperty("idle_timeout_ms", out var itElement) && itElement.ValueKind == JsonValueKind.Number ? itElement.GetInt32() : 0, + PrefixPaddingInMilliseconds = turnDetectionElement.TryGetProperty("prefix_padding_ms", out var ppElement) && ppElement.ValueKind == JsonValueKind.Number ? ppElement.GetInt32() : 300, + SilenceDurationInMilliseconds = turnDetectionElement.TryGetProperty("silence_duration_ms", out var sdElement) && sdElement.ValueKind == JsonValueKind.Number ? sdElement.GetInt32() : 500, + Threshold = turnDetectionElement.TryGetProperty("threshold", out var thElement) && thElement.ValueKind == JsonValueKind.Number ? thElement.GetDouble() : 0.5, + }; + } + + if (vadType == "semantic_vad") + { + return new SemanticVoiceActivityDetection + { + CreateResponse = turnDetectionElement.TryGetProperty("create_response", out var crElement) && crElement.GetBoolean(), + InterruptResponse = turnDetectionElement.TryGetProperty("interrupt_response", out var irElement) && irElement.GetBoolean(), + Eagerness = turnDetectionElement.TryGetProperty("eagerness", out var eagernessElement) && eagernessElement.GetString() is string eagerness + ? new SemanticEagerness(eagerness) + : SemanticEagerness.Auto, + }; + } + + return null; } private static RealtimeServerErrorMessage? CreateErrorMessage(JsonElement root) diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs index 87ca8f7efdf..899843fbaf1 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs @@ -341,7 +341,7 @@ private static bool ExtractFunctionCalls(RealtimeServerResponseOutputItemMessage /// private bool ShouldTerminateBasedOnFunctionCalls(List functionCallContents) { - var (toolMap, _) = FunctionInvocationHelpers.CreateToolsMap(AdditionalTools, InnerSession.Options?.Tools); + var (toolMap, _) = FunctionInvocationHelpers.CreateToolsMap(AdditionalTools, InnerSession.Options?.Tools as IList); if (toolMap is null || toolMap.Count == 0) { @@ -389,7 +389,7 @@ private bool ShouldTerminateBasedOnFunctionCalls(List funct CancellationToken cancellationToken) { // Compute toolMap to ensure we always use the latest tools - var (toolMap, _) = FunctionInvocationHelpers.CreateToolsMap(AdditionalTools, InnerSession.Options?.Tools); + var (toolMap, _) = FunctionInvocationHelpers.CreateToolsMap(AdditionalTools, InnerSession.Options?.Tools as IList); var captureCurrentIterationExceptions = consecutiveErrorCount < MaximumConsecutiveErrorsPerRequest; diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs index c7ed4204248..2e50aba62c4 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs @@ -34,8 +34,6 @@ public void Constructor_Parameterless_PropsDefaulted() [Fact] public void Properties_Roundtrip() { - RealtimeSessionOptions options = new(); - var inputFormat = new RealtimeAudioFormat("audio/pcm", 16000); var outputFormat = new RealtimeAudioFormat("audio/pcm", 24000); List modalities = ["text", "audio"]; @@ -43,20 +41,23 @@ public void Properties_Roundtrip() var transcriptionOptions = new TranscriptionOptions { SpeechLanguage = "en", ModelId = "whisper-1", Prompt = "greeting" }; var vad = new VoiceActivityDetection { CreateResponse = true, InterruptResponse = true }; - options.SessionKind = RealtimeSessionKind.Transcription; - options.Model = "gpt-4-realtime"; - options.InputAudioFormat = inputFormat; - options.OutputAudioFormat = outputFormat; - options.NoiseReductionOptions = NoiseReductionOptions.NearField; - options.TranscriptionOptions = transcriptionOptions; - options.VoiceActivityDetection = vad; - options.VoiceSpeed = 1.5; - options.Voice = "alloy"; - options.Instructions = "Be helpful"; - options.MaxOutputTokens = 500; - options.OutputModalities = modalities; - options.ToolMode = ChatToolMode.Auto; - options.Tools = tools; + RealtimeSessionOptions options = new() + { + SessionKind = RealtimeSessionKind.Transcription, + Model = "gpt-4-realtime", + InputAudioFormat = inputFormat, + OutputAudioFormat = outputFormat, + NoiseReductionOptions = NoiseReductionOptions.NearField, + TranscriptionOptions = transcriptionOptions, + VoiceActivityDetection = vad, + VoiceSpeed = 1.5, + Voice = "alloy", + Instructions = "Be helpful", + MaxOutputTokens = 500, + OutputModalities = modalities, + ToolMode = ChatToolMode.Auto, + Tools = tools, + }; Assert.Equal(RealtimeSessionKind.Transcription, options.SessionKind); Assert.Equal("gpt-4-realtime", options.Model); @@ -107,10 +108,9 @@ public void VoiceActivityDetection_Properties_Roundtrip() Assert.False(vad.CreateResponse); Assert.False(vad.InterruptResponse); - vad.CreateResponse = true; - vad.InterruptResponse = true; + var vad2 = new VoiceActivityDetection { CreateResponse = true, InterruptResponse = true }; - Assert.True(vad.CreateResponse); - Assert.True(vad.InterruptResponse); + Assert.True(vad2.CreateResponse); + Assert.True(vad2.InterruptResponse); } } From aa17989732890702ed3f6873416f4e7c01dfbcec Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Mon, 23 Feb 2026 13:01:52 -0800 Subject: [PATCH 38/92] Fix XML doc param order, add null validation to constructors --- .../Realtime/RealtimeClientConversationItemCreateMessage.cs | 3 ++- .../Realtime/RealtimeClientInputAudioBufferAppendMessage.cs | 3 ++- .../Realtime/RealtimeContentItem.cs | 2 +- .../Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs index 979e76f687b..dc10766e347 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs @@ -3,6 +3,7 @@ using System.Diagnostics.CodeAnalysis; using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; @@ -20,7 +21,7 @@ public class RealtimeClientConversationItemCreateMessage : RealtimeClientMessage public RealtimeClientConversationItemCreateMessage(RealtimeContentItem item, string? previousId = null) { PreviousId = previousId; - Item = item; + Item = Throw.IfNull(item); } /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs index 2e1f9c998d2..b1b9dd2f038 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs @@ -3,6 +3,7 @@ using System.Diagnostics.CodeAnalysis; using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; @@ -19,7 +20,7 @@ public class RealtimeClientInputAudioBufferAppendMessage : RealtimeClientMessage /// The data content containing the audio buffer data to append. public RealtimeClientInputAudioBufferAppendMessage(DataContent audioContent) { - Content = audioContent; + Content = Throw.IfNull(audioContent); } /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs index 9f3040dc326..f6d6d7e9c39 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs @@ -20,9 +20,9 @@ public class RealtimeContentItem /// /// Initializes a new instance of the class. /// + /// The contents of the conversation item. /// The ID of the conversation item. /// The role of the conversation item. - /// The contents of the conversation item. public RealtimeContentItem(IList contents, string? id = null, ChatRole? role = null) { Id = id; diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 52571623ead..520b117b0a3 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -64,8 +64,8 @@ public sealed class OpenAIRealtimeSession : IRealtimeSession /// The model to use for the session. public OpenAIRealtimeSession(string apiKey, string model) { - _apiKey = apiKey; - _model = model; + _apiKey = Throw.IfNull(apiKey); + _model = Throw.IfNull(model); _eventChannel = Channel.CreateUnbounded(); } From c7dbcf825e994f3ce563126c84b6b1e515696e72 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 3 Mar 2026 14:36:15 -0800 Subject: [PATCH 39/92] OpenAI Realtime Provider using OpenAI SDK --- .../OpenAIRealtimeClient.cs | 47 +- .../OpenAIRealtimeSession.cs | 2309 +++++++---------- .../OpenAIRealtimeClientTests.cs | 2 +- 3 files changed, 927 insertions(+), 1431 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs index a543fc6526b..efd55da313e 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs @@ -3,12 +3,16 @@ using System; using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Net.WebSockets; using System.Threading; using System.Threading.Tasks; using Microsoft.Shared.DiagnosticIds; using Microsoft.Shared.Diagnostics; +using OpenAI.Realtime; #pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. +#pragma warning disable OPENAI002 // OpenAI Realtime API is experimental namespace Microsoft.Extensions.AI; @@ -16,12 +20,15 @@ namespace Microsoft.Extensions.AI; [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] public sealed class OpenAIRealtimeClient : IRealtimeClient { - /// The API key used for authentication. - private readonly string _apiKey; + /// The OpenAI Realtime client. + private readonly RealtimeClient _realtimeClient; /// The model to use for realtime sessions. private readonly string _model; + /// Metadata about this client's provider and model, used for OpenTelemetry. + private readonly ChatClientMetadata _metadata; + /// Initializes a new instance of the class. /// The API key used for authentication. /// The model to use for realtime sessions. @@ -29,23 +36,41 @@ public sealed class OpenAIRealtimeClient : IRealtimeClient /// is . public OpenAIRealtimeClient(string apiKey, string model) { - _apiKey = Throw.IfNull(apiKey); + _realtimeClient = new RealtimeClient(Throw.IfNull(apiKey)); + _model = Throw.IfNull(model); + _metadata = new("openai", defaultModelId: _model); + } + + /// Initializes a new instance of the class. + /// The OpenAI Realtime client to use. + /// The model to use for realtime sessions. + /// is . + /// is . + public OpenAIRealtimeClient(RealtimeClient realtimeClient, string model) + { + _realtimeClient = Throw.IfNull(realtimeClient); _model = Throw.IfNull(model); + _metadata = new("openai", defaultModelId: _model); } /// public async Task CreateSessionAsync(RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default) { - var session = new OpenAIRealtimeSession(_apiKey, _model); + RealtimeSessionClient sessionClient; try { - bool connected = await session.ConnectAsync(cancellationToken).ConfigureAwait(false); - if (!connected) - { - await session.DisposeAsync().ConfigureAwait(false); - return null; - } + sessionClient = options?.SessionKind == RealtimeSessionKind.Transcription + ? await _realtimeClient.StartTranscriptionSessionAsync(cancellationToken: cancellationToken).ConfigureAwait(false) + : await _realtimeClient.StartConversationSessionAsync(_model, cancellationToken: cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is WebSocketException or OperationCanceledException or IOException) + { + return null; + } + var session = new OpenAIRealtimeSession(sessionClient, _model); + try + { if (options is not null) { await session.UpdateAsync(options, cancellationToken).ConfigureAwait(false); @@ -67,7 +92,9 @@ public OpenAIRealtimeClient(string apiKey, string model) return serviceKey is not null ? null : + serviceType == typeof(ChatClientMetadata) ? _metadata : serviceType.IsInstanceOfType(this) ? this : + serviceType.IsInstanceOfType(_realtimeClient) ? _realtimeClient : null; } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 520b117b0a3..380cea2bdeb 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -2,23 +2,22 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.Buffers; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using System.Net.WebSockets; using System.Runtime.CompilerServices; -using System.Text; using System.Text.Json; using System.Text.Json.Nodes; using System.Threading; -using System.Threading.Channels; using System.Threading.Tasks; using Microsoft.Shared.DiagnosticIds; using Microsoft.Shared.Diagnostics; +using Sdk = OpenAI.Realtime; #pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. +#pragma warning disable OPENAI002 // OpenAI Realtime API is experimental #pragma warning disable SA1204 // Static elements should appear before instance elements #pragma warning disable IL2026 // Members annotated with 'RequiresUnreferencedCodeAttribute' require dynamic access #pragma warning disable IL3050 // Members annotated with 'RequiresDynamicCodeAttribute' require dynamic access @@ -29,33 +28,21 @@ namespace Microsoft.Extensions.AI; [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] public sealed class OpenAIRealtimeSession : IRealtimeSession { - /// The receive buffer size for WebSocket messages. - private const int ReceiveBufferSize = 1024 * 16; - - /// The API key used for authentication. - private readonly string _apiKey; - /// The model to use for the session. private readonly string _model; - /// Lock used to synchronize WebSocket send operations. - private readonly SemaphoreSlim _sendLock = new(1, 1); + /// Metadata about this session's provider and model, used for OpenTelemetry. + private readonly ChatClientMetadata _metadata; - /// Channel used to relay parsed server events to the consumer. - private readonly Channel _eventChannel; + /// Owned created from the (apiKey, model) constructor path. + private Sdk.RealtimeClient? _ownedRealtimeClient; - /// The WebSocket used for communication. - private WebSocket? _webSocket; - - /// The cancellation token source for the session. - private CancellationTokenSource? _cancellationTokenSource; + /// The SDK session client for communication with the Realtime API. + private Sdk.RealtimeSessionClient? _sessionClient; /// Whether the session has been disposed (0 = false, 1 = true). private int _disposed; - /// The background task that receives WebSocket messages. - private Task? _receiveTask; - /// public RealtimeSessionOptions? Options { get; private set; } @@ -64,9 +51,19 @@ public sealed class OpenAIRealtimeSession : IRealtimeSession /// The model to use for the session. public OpenAIRealtimeSession(string apiKey, string model) { - _apiKey = Throw.IfNull(apiKey); + _ownedRealtimeClient = new Sdk.RealtimeClient(Throw.IfNull(apiKey)); _model = Throw.IfNull(model); - _eventChannel = Channel.CreateUnbounded(); + _metadata = new("openai", defaultModelId: _model); + } + + /// Initializes a new instance of the class from an already-connected session client. + /// The connected SDK session client. + /// The model name for metadata. + internal OpenAIRealtimeSession(Sdk.RealtimeSessionClient sessionClient, string model) + { + _sessionClient = Throw.IfNull(sessionClient); + _model = model ?? string.Empty; + _metadata = new("openai", defaultModelId: _model); } /// Connects the WebSocket to the OpenAI Realtime API. @@ -74,23 +71,15 @@ public OpenAIRealtimeSession(string apiKey, string model) /// if the connection succeeded; otherwise, . public async Task ConnectAsync(CancellationToken cancellationToken = default) { - try + if (_ownedRealtimeClient is null) { - var clientWebSocket = new ClientWebSocket(); - clientWebSocket.Options.SetRequestHeader("Authorization", $"Bearer {_apiKey}"); - _webSocket = clientWebSocket; - - // Use an independent CTS for the session lifetime. The creation-time token is only - // used for the connect call itself; the background receive loop must not be tied to it, - // since callers commonly use short-lived timeout tokens for creation. - _cancellationTokenSource = new CancellationTokenSource(); - - var uri = new Uri($"wss://api.openai.com/v1/realtime?model={Uri.EscapeDataString(_model)}"); - await clientWebSocket.ConnectAsync(uri, cancellationToken).ConfigureAwait(false); - - // Start receiving messages in the background. - _receiveTask = Task.Run(() => ReceiveMessagesAsync(_cancellationTokenSource.Token), _cancellationTokenSource.Token); + return false; + } + try + { + _sessionClient = await _ownedRealtimeClient.StartConversationSessionAsync( + _model, cancellationToken: cancellationToken).ConfigureAwait(false); return true; } catch (Exception ex) when (ex is WebSocketException or OperationCanceledException or IOException) @@ -99,175 +88,37 @@ public async Task ConnectAsync(CancellationToken cancellationToken = defau } } - /// Connects the session using an already-connected instance. Used for testing. - /// The connected WebSocket to use. - /// The to monitor for cancellation requests. - internal Task ConnectWithWebSocketAsync(WebSocket webSocket, CancellationToken cancellationToken) - { - _webSocket = webSocket; - _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - - // Start receiving messages in the background. - _receiveTask = Task.Run(() => ReceiveMessagesAsync(_cancellationTokenSource.Token), _cancellationTokenSource.Token); - - return Task.CompletedTask; - } - /// public async Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancellationToken = default) { _ = Throw.IfNull(options); - // Note: When switching to the OpenAI SDK for serialization, consume options.RawRepresentationFactory - // here to allow callers to provide a pre-configured SDK-specific options instance, following the - // same pattern used by OpenAIChatClient and other provider implementations. - - var sessionElement = new JsonObject + if (_sessionClient is not null) { - ["type"] = "session.update", - }; - var sessionObject = new JsonObject(); - sessionElement["session"] = sessionObject; - - var audioElement = new JsonObject(); - var audioInputElement = new JsonObject(); - var audioOutputElement = new JsonObject(); - sessionObject["audio"] = audioElement; - audioElement["input"] = audioInputElement; - audioElement["output"] = audioOutputElement; + // Allow callers to provide a pre-configured SDK-specific options instance. + object? rawOptions = options.RawRepresentationFactory?.Invoke(this); - if (options.InputAudioFormat is not null) - { - var audioInputFormatElement = new JsonObject - { - ["type"] = options.InputAudioFormat.MediaType, - }; - if (options.InputAudioFormat.SampleRate.HasValue) + if (rawOptions is Sdk.RealtimeConversationSessionOptions rawConvOptions) { - audioInputFormatElement["rate"] = options.InputAudioFormat.SampleRate.Value; + await _sessionClient.ConfigureConversationSessionAsync(rawConvOptions, cancellationToken).ConfigureAwait(false); } - - audioInputElement["format"] = audioInputFormatElement; - } - - if (options.NoiseReductionOptions.HasValue) - { - var noiseReductionObj = new JsonObject - { - ["type"] = options.NoiseReductionOptions.Value == NoiseReductionOptions.NearField ? "near_field" : "far_field", - }; - audioInputElement["noise_reduction"] = noiseReductionObj; - } - - if (options.TranscriptionOptions is not null) - { - var transcriptionOptionsObj = new JsonObject + else if (rawOptions is Sdk.RealtimeTranscriptionSessionOptions rawTransOptions) { - ["language"] = options.TranscriptionOptions.SpeechLanguage, - ["model"] = options.TranscriptionOptions.ModelId, - }; - if (options.TranscriptionOptions.Prompt is not null) - { - transcriptionOptionsObj["prompt"] = options.TranscriptionOptions.Prompt; + await _sessionClient.ConfigureTranscriptionSessionAsync(rawTransOptions, cancellationToken).ConfigureAwait(false); } - - audioInputElement["transcription"] = transcriptionOptionsObj; - } - - if (options.VoiceActivityDetection is ServerVoiceActivityDetection serverVad) - { - audioInputElement["turn_detection"] = new JsonObject + else if (options.SessionKind == RealtimeSessionKind.Transcription) { - ["type"] = "server_vad", - ["create_response"] = serverVad.CreateResponse, - ["idle_timeout_ms"] = serverVad.IdleTimeoutInMilliseconds, - ["interrupt_response"] = serverVad.InterruptResponse, - ["prefix_padding_ms"] = serverVad.PrefixPaddingInMilliseconds, - ["silence_duration_ms"] = serverVad.SilenceDurationInMilliseconds, - ["threshold"] = serverVad.Threshold, - }; - } - else if (options.VoiceActivityDetection is SemanticVoiceActivityDetection semanticVad) - { - audioInputElement["turn_detection"] = new JsonObject - { - ["type"] = "semantic_vad", - ["create_response"] = semanticVad.CreateResponse, - ["interrupt_response"] = semanticVad.InterruptResponse, - ["eagerness"] = semanticVad.Eagerness.Value, - }; - } - - if (options.SessionKind == RealtimeSessionKind.Realtime) - { - sessionObject["type"] = "realtime"; - - if (options.OutputAudioFormat is not null) - { - var audioOutputFormatElement = new JsonObject - { - ["type"] = options.OutputAudioFormat.MediaType, - }; - if (options.OutputAudioFormat.SampleRate.HasValue) - { - audioOutputFormatElement["rate"] = options.OutputAudioFormat.SampleRate.Value; - } - - audioOutputElement["format"] = audioOutputFormatElement; + var transOpts = BuildTranscriptionSessionOptions(options); + await _sessionClient.ConfigureTranscriptionSessionAsync(transOpts, cancellationToken).ConfigureAwait(false); } - - audioOutputElement["speed"] = options.VoiceSpeed; - - if (options.Voice is not null) - { - audioOutputElement["voice"] = options.Voice; - } - - if (options.Instructions is not null) - { - sessionObject["instructions"] = options.Instructions; - } - - if (options.MaxOutputTokens.HasValue) - { - sessionObject["max_output_tokens"] = options.MaxOutputTokens.Value; - } - - if (options.Model is not null) - { - sessionObject["model"] = options.Model; - } - - if (options.OutputModalities is not null && options.OutputModalities.Any()) - { - sessionObject["output_modalities"] = CreateModalitiesArray(options.OutputModalities); - } - - if (options.Tools is not null) + else { - var toolsArray = new JsonArray(); - foreach (var tool in options.Tools) - { - JsonObject? toolObj = tool is HostedMcpServerTool mcpTool - ? SerializeMcpToolToJson(mcpTool) - : SerializeAIFunctionToJson(tool as AIFunction); - if (toolObj is not null) - { - toolsArray.Add(toolObj); - } - } - - sessionObject["tools"] = toolsArray; + var convOpts = BuildConversationSessionOptions(options); + await _sessionClient.ConfigureConversationSessionAsync(convOpts, cancellationToken).ConfigureAwait(false); } } - else if (options.SessionKind == RealtimeSessionKind.Transcription) - { - sessionObject["type"] = "transcription"; - } Options = options; - - await SendEventAsync(sessionElement, cancellationToken).ConfigureAwait(false); } /// @@ -275,298 +126,48 @@ public async Task SendClientMessageAsync(RealtimeClientMessage message, Cancella { _ = Throw.IfNull(message); - // In a realtime streaming scenario, cancellation is used to signal session teardown. - // Silently returning avoids throwing during the natural shutdown path, where multiple - // components may race to inject final messages while the session is being disposed. - if (cancellationToken.IsCancellationRequested) + if (cancellationToken.IsCancellationRequested || _sessionClient is null) { return; } - JsonObject? jsonMessage = new JsonObject(); - - if (message.MessageId is not null) - { - jsonMessage["event_id"] = message.MessageId; - } - - switch (message) + try { - case RealtimeClientResponseCreateMessage responseCreate: - jsonMessage["type"] = "response.create"; - var responseObj = new JsonObject(); - - // Handle OutputAudioOptions (audio.output.format). - if (responseCreate.OutputAudioOptions is not null) - { - var audioObj = new JsonObject(); - var outputObj = new JsonObject(); - var formatObj = new JsonObject(); - - switch (responseCreate.OutputAudioOptions.MediaType) - { - case "audio/pcm": - if (responseCreate.OutputAudioOptions.SampleRate == 24000) - { - formatObj["type"] = responseCreate.OutputAudioOptions.MediaType; - formatObj["rate"] = 24000; - } - - break; - - case "audio/pcmu": - case "audio/pcma": - formatObj["type"] = responseCreate.OutputAudioOptions.MediaType; - break; - } - - outputObj["format"] = formatObj; - - if (!string.IsNullOrEmpty(responseCreate.OutputVoice)) - { - outputObj["voice"] = responseCreate.OutputVoice; - } - - audioObj["output"] = outputObj; - responseObj["audio"] = audioObj; - } - else if (!string.IsNullOrEmpty(responseCreate.OutputVoice)) - { - responseObj["audio"] = new JsonObject - { - ["output"] = new JsonObject - { - ["voice"] = responseCreate.OutputVoice, - }, - }; - } - - responseObj["conversation"] = responseCreate.ExcludeFromConversation ? "none" : "auto"; - - if (responseCreate.Items is { } items && items.Any()) - { - var inputArray = new JsonArray(); - foreach (var item in items) - { - if (item is RealtimeContentItem contentItem && contentItem.Contents is not null) - { - var itemObj = new JsonObject - { - ["type"] = "message", - ["object"] = "realtime.item", - }; - - if (contentItem.Role.HasValue) - { - itemObj["role"] = contentItem.Role.Value.Value; - } - - if (contentItem.Id is not null) - { - itemObj["id"] = contentItem.Id; - } - - itemObj["content"] = SerializeContentsToJsonArray(contentItem.Contents); - inputArray.Add(itemObj); - } - } - - responseObj["input"] = inputArray; - } - - if (!string.IsNullOrEmpty(responseCreate.Instructions)) - { - responseObj["instructions"] = responseCreate.Instructions; - } - - if (responseCreate.MaxOutputTokens.HasValue) - { - responseObj["max_output_tokens"] = responseCreate.MaxOutputTokens.Value; - } - - if (responseCreate.AdditionalProperties is { Count: > 0 }) - { - var metadataObj = new JsonObject(); - foreach (var kvp in responseCreate.AdditionalProperties) - { - metadataObj[kvp.Key] = JsonValue.Create(kvp.Value); - } - - responseObj["metadata"] = metadataObj; - } - - if (responseCreate.OutputModalities is not null && responseCreate.OutputModalities.Any()) - { - responseObj["output_modalities"] = CreateModalitiesArray(responseCreate.OutputModalities); - } - - if (responseCreate.ToolMode is { } toolMode) - { - responseObj["tool_choice"] = toolMode switch - { - RequiredChatToolMode r when r.RequiredFunctionName is not null => new JsonObject - { - ["type"] = "function", - ["name"] = r.RequiredFunctionName, - }, - RequiredChatToolMode => "required", - NoneChatToolMode => "none", - _ => "auto", - }; - } - - if (responseCreate.Tools is not null && responseCreate.Tools.Any()) - { - var toolsArray = new JsonArray(); - - foreach (var tool in responseCreate.Tools) - { - JsonObject? toolObj = null; - - if (tool is AIFunction aiFunction && !string.IsNullOrEmpty(aiFunction.Name)) - { - toolObj = SerializeAIFunctionToJson(aiFunction); - } - else if (tool is HostedMcpServerTool mcpTool) - { - toolObj = SerializeMcpToolToJson(mcpTool); - } - else - { - var toolJson = JsonSerializer.SerializeToNode(tool); - - if (toolJson is not null && - (toolJson["server_label"] is not null || toolJson["server_url"] is not null || toolJson["connector_id"] is not null)) - { - toolObj = new JsonObject { ["type"] = "mcp" }; - - CopyJsonPropertyIfExists(toolJson, toolObj, "server_label"); - CopyJsonPropertyIfExists(toolJson, toolObj, "server_url"); - CopyJsonPropertyIfExists(toolJson, toolObj, "connector_id"); - CopyJsonPropertyIfExists(toolJson, toolObj, "authorization"); - CopyJsonPropertyIfExists(toolJson, toolObj, "headers"); - CopyJsonPropertyIfExists(toolJson, toolObj, "require_approval"); - CopyJsonPropertyIfExists(toolJson, toolObj, "server_description"); - CopyJsonPropertyIfExists(toolJson, toolObj, "allowed_tools"); - } - } - - if (toolObj is not null) - { - toolsArray.Add(toolObj); - } - } - - responseObj["tools"] = toolsArray; - } + switch (message) + { + case RealtimeClientResponseCreateMessage responseCreate: + await SendResponseCreateAsync(responseCreate, cancellationToken).ConfigureAwait(false); + break; - jsonMessage["response"] = responseObj; - break; + case RealtimeClientConversationItemCreateMessage itemCreate: + await SendConversationItemCreateAsync(itemCreate, cancellationToken).ConfigureAwait(false); + break; - case RealtimeClientConversationItemCreateMessage itemCreate: - if (itemCreate.Item is not null) - { - jsonMessage["type"] = "conversation.item.create"; + case RealtimeClientInputAudioBufferAppendMessage audioAppend: + await SendInputAudioAppendAsync(audioAppend, cancellationToken).ConfigureAwait(false); + break; - if (itemCreate.PreviousId is not null) + case RealtimeClientInputAudioBufferCommitMessage: + if (message.MessageId is not null) { - jsonMessage["previous_item_id"] = itemCreate.PreviousId; + var cmd = new Sdk.RealtimeClientCommandInputAudioBufferCommit { EventId = message.MessageId }; + await _sessionClient.SendCommandAsync(cmd, cancellationToken).ConfigureAwait(false); } - - if (itemCreate.Item is RealtimeContentItem contentItem && contentItem.Contents is not null) + else { - var itemObj = new JsonObject(); - - if (contentItem.Id is not null) - { - itemObj["id"] = contentItem.Id; - } - - if (contentItem.Contents.Count > 0 && contentItem.Contents[0] is FunctionResultContent functionResult) - { - itemObj["type"] = "function_call_output"; - itemObj["call_id"] = functionResult.CallId; - itemObj["output"] = functionResult?.Result is not null - ? JsonSerializer.Serialize(functionResult.Result) - : null; - } - else if (contentItem.Contents.Count > 0 && contentItem.Contents[0] is FunctionCallContent functionCall) - { - itemObj["type"] = "function_call"; - itemObj["call_id"] = functionCall.CallId; - itemObj["name"] = functionCall.Name; - - if (functionCall.Arguments is not null) - { - itemObj["arguments"] = JsonSerializer.Serialize(functionCall.Arguments); - } - } - else if (contentItem.Contents.Count > 0 && contentItem.Contents[0] is McpServerToolApprovalResponseContent approvalResponse) - { - itemObj["type"] = "mcp_approval_response"; - itemObj["approval_request_id"] = approvalResponse.Id; - itemObj["approve"] = approvalResponse.Approved; - } - else - { - itemObj["type"] = "message"; - - if (contentItem.Role.HasValue) - { - itemObj["role"] = contentItem.Role.Value.Value; - } - - itemObj["content"] = SerializeContentsToJsonArray(contentItem.Contents); - } - - jsonMessage["item"] = itemObj; + await _sessionClient.CommitPendingAudioAsync(cancellationToken).ConfigureAwait(false); } - } - break; - - case RealtimeClientInputAudioBufferAppendMessage audioAppend: - if (audioAppend.Content is not null && audioAppend.Content.HasTopLevelMediaType("audio")) - { - jsonMessage["type"] = "input_audio_buffer.append"; - - string dataUri = audioAppend.Content.Uri?.ToString() ?? string.Empty; - int commaIndex = dataUri.LastIndexOf(','); - - jsonMessage["audio"] = commaIndex >= 0 && commaIndex < dataUri.Length - 1 - ? dataUri.Substring(commaIndex + 1) - : Convert.ToBase64String(audioAppend.Content.Data.ToArray()); - } - - break; - - case RealtimeClientInputAudioBufferCommitMessage: - jsonMessage["type"] = "input_audio_buffer.commit"; - break; - - default: - if (message.RawRepresentation is string rawString) - { - jsonMessage = JsonSerializer.Deserialize(rawString); - } - else if (message.RawRepresentation is JsonObject rawJsonObject) - { - jsonMessage = rawJsonObject; - } - - // Preserve MessageId if it was set on the message but not in the raw representation. - if (jsonMessage is not null && message.MessageId is not null && - !jsonMessage.ContainsKey("event_id")) - { - jsonMessage["event_id"] = message.MessageId; - } + break; - break; + default: + await SendRawCommandAsync(message, cancellationToken).ConfigureAwait(false); + break; + } } - - if (jsonMessage?.TryGetPropertyValue("type", out _) is true) + catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException or WebSocketException) { - await SendEventAsync(jsonMessage, cancellationToken).ConfigureAwait(false); + // Expected during session teardown or cancellation. } } @@ -574,9 +175,18 @@ public async Task SendClientMessageAsync(RealtimeClientMessage message, Cancella public async IAsyncEnumerable GetStreamingResponseAsync( [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (var serverEvent in _eventChannel.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + if (_sessionClient is null) + { + yield break; + } + + await foreach (var update in _sessionClient.ReceiveUpdatesAsync(cancellationToken).ConfigureAwait(false)) { - yield return serverEvent; + var serverMessage = MapServerUpdate(update); + if (serverMessage is not null) + { + yield return serverMessage; + } } } @@ -587,7 +197,9 @@ public async IAsyncEnumerable GetStreamingResponseAsync( return serviceKey is not null ? null : + serviceType == typeof(ChatClientMetadata) ? _metadata : serviceType.IsInstanceOfType(this) ? this : + _sessionClient is not null && serviceType.IsInstanceOfType(_sessionClient) ? _sessionClient : null; } @@ -599,1326 +211,1183 @@ public void Dispose() return; } - _ = _eventChannel.Writer.TryComplete(); - try - { - _cancellationTokenSource?.Cancel(); - } - catch (ObjectDisposedException) - { - // Already disposed; nothing to do. - } - - // Best-effort wait for the background receive loop to finish. -#pragma warning disable VSTHRD002 // Synchronous Dispose must block; callers should prefer DisposeAsync - try - { - _receiveTask?.GetAwaiter().GetResult(); - } - catch (OperationCanceledException) - { - // Expected when the session is cancelled during disposal. - } -#pragma warning restore VSTHRD002 - - _cancellationTokenSource?.Dispose(); - _webSocket?.Dispose(); - _sendLock.Dispose(); + _sessionClient?.Dispose(); } /// - public async ValueTask DisposeAsync() + public ValueTask DisposeAsync() { if (Interlocked.Exchange(ref _disposed, 1) != 0) { - return; + return default; } - _ = _eventChannel.Writer.TryComplete(); - try - { -#if NET - await (_cancellationTokenSource?.CancelAsync() ?? Task.CompletedTask).ConfigureAwait(false); -#else - _cancellationTokenSource?.Cancel(); -#endif - } - catch (ObjectDisposedException) - { - // Already disposed; nothing to do. - } + _sessionClient?.Dispose(); + return default; + } - // Wait for the background receive loop to finish before disposing resources. - if (_receiveTask is not null) + #region Send Helpers (MEAI → SDK) + + private async Task SendResponseCreateAsync(RealtimeClientResponseCreateMessage responseCreate, CancellationToken cancellationToken) + { + var responseOptions = new Sdk.RealtimeResponseOptions(); + + // Audio output options. + if (responseCreate.OutputAudioOptions is not null || !string.IsNullOrEmpty(responseCreate.OutputVoice)) { - try + responseOptions.AudioOptions = new Sdk.RealtimeResponseAudioOptions(); + if (responseCreate.OutputAudioOptions is not null) { - await _receiveTask.ConfigureAwait(false); + responseOptions.AudioOptions.OutputAudioOptions.AudioFormat = ToSdkAudioFormat(responseCreate.OutputAudioOptions); } - catch (OperationCanceledException) + + if (!string.IsNullOrEmpty(responseCreate.OutputVoice)) { - // Expected when the session is cancelled during disposal. + responseOptions.AudioOptions.OutputAudioOptions.Voice = new Sdk.RealtimeVoice(responseCreate.OutputVoice); } } - _cancellationTokenSource?.Dispose(); - _webSocket?.Dispose(); - _sendLock.Dispose(); - } + // Conversation mode. + responseOptions.DefaultConversationConfiguration = responseCreate.ExcludeFromConversation + ? Sdk.RealtimeResponseDefaultConversationConfiguration.None + : Sdk.RealtimeResponseDefaultConversationConfiguration.Auto; - /// Sends a JSON event over the WebSocket. - /// The JSON object to send. - /// The to monitor for cancellation requests. - private async Task SendEventAsync(JsonObject eventData, CancellationToken cancellationToken) - { - if (Volatile.Read(ref _disposed) != 0 || _webSocket?.State != WebSocketState.Open) + // Input items. + if (responseCreate.Items is { } items) { - return; - } - - try - { - var utf8Bytes = JsonSerializer.SerializeToUtf8Bytes(eventData); - var lockTaken = false; - try + foreach (var item in items) { - await _sendLock.WaitAsync(cancellationToken).ConfigureAwait(false); - lockTaken = true; - - await _webSocket.SendAsync( - new ArraySegment(utf8Bytes), - WebSocketMessageType.Text, - true, - cancellationToken).ConfigureAwait(false); - } - finally - { - if (lockTaken) + if (ToRealtimeItem(item) is Sdk.RealtimeItem sdkItem) { - _ = _sendLock.Release(); + responseOptions.InputItems.Add(sdkItem); } } } - catch (OperationCanceledException) - { - // Expected when the session is cancelled. - } - catch (ObjectDisposedException) + + if (!string.IsNullOrEmpty(responseCreate.Instructions)) { - // Expected when the session is disposed concurrently. + responseOptions.Instructions = responseCreate.Instructions; } - catch (WebSocketException) + + if (responseCreate.MaxOutputTokens.HasValue) { - // Expected when the WebSocket is in an aborted state. + responseOptions.MaxOutputTokenCount = responseCreate.MaxOutputTokens.Value; } - } - /// Background loop that receives WebSocket messages and writes them to the event channel. - /// The to monitor for cancellation requests. - private async Task ReceiveMessagesAsync(CancellationToken cancellationToken) - { - byte[] buffer = ArrayPool.Shared.Rent(ReceiveBufferSize); - try + if (responseCreate.AdditionalProperties is { Count: > 0 }) { - while (_webSocket?.State == WebSocketState.Open && !cancellationToken.IsCancellationRequested) + var metadata = new Dictionary(); + foreach (var kvp in responseCreate.AdditionalProperties) { - var result = await _webSocket.ReceiveAsync(new ArraySegment(buffer), cancellationToken).ConfigureAwait(false); - - if (result.MessageType == WebSocketMessageType.Close) - { - break; - } - - string message; - if (result.EndOfMessage) - { - // Fast path: single-frame message, no extra allocation. - message = Encoding.UTF8.GetString(buffer, 0, result.Count); - } - else - { - // Multi-frame: accumulate remaining frames. - using var ms = new MemoryStream(); -#if NET - await ms.WriteAsync(buffer.AsMemory(0, result.Count), cancellationToken).ConfigureAwait(false); -#else - await ms.WriteAsync(buffer, 0, result.Count, cancellationToken).ConfigureAwait(false); -#endif - do - { - result = await _webSocket.ReceiveAsync(new ArraySegment(buffer), cancellationToken).ConfigureAwait(false); - if (result.MessageType == WebSocketMessageType.Close) - { - break; - } - -#if NET - await ms.WriteAsync(buffer.AsMemory(0, result.Count), cancellationToken).ConfigureAwait(false); -#else - await ms.WriteAsync(buffer, 0, result.Count, cancellationToken).ConfigureAwait(false); -#endif - } - while (!result.EndOfMessage); - - if (result.MessageType == WebSocketMessageType.Close) - { - break; - } - - message = Encoding.UTF8.GetString(ms.GetBuffer(), 0, (int)ms.Length); - } - - if (result.MessageType == WebSocketMessageType.Text) - { - ProcessServerEvent(message); - } + metadata[kvp.Key] = BinaryData.FromString(kvp.Value?.ToString() ?? string.Empty); } + + responseOptions.Metadata = metadata; } - catch (OperationCanceledException) - { - // Expected when the session is cancelled. - } - catch (Exception ex) + + if (responseCreate.OutputModalities is not null) { - // Surface the error to consumers reading from the channel. - _ = _eventChannel.Writer.TryComplete(ex); + foreach (var modality in responseCreate.OutputModalities) + { + responseOptions.OutputModalities.Add(new Sdk.RealtimeOutputModality(modality)); + } } - finally + + if (responseCreate.ToolMode is { } toolMode) { - ArrayPool.Shared.Return(buffer); - _ = _eventChannel.Writer.TryComplete(); + responseOptions.ToolChoice = ToSdkToolChoice(toolMode); } - } - /// Parses a raw JSON server event and writes the corresponding to the channel. - /// The raw JSON message string. - private void ProcessServerEvent(string message) - { - try + if (responseCreate.Tools is not null) { - using var jsonDoc = JsonDocument.Parse(message); - var root = jsonDoc.RootElement; - var eventType = root.GetProperty("type").GetString(); - - try + foreach (var tool in responseCreate.Tools) { - var serverMessage = eventType switch + if (ToRealtimeTool(tool) is Sdk.RealtimeTool sdkTool) { - "error" => CreateErrorMessage(root), - "conversation.item.input_audio_transcription.delta" or - "conversation.item.input_audio_transcription.completed" or - "conversation.item.input_audio_transcription.failed" => - CreateInputAudioTranscriptionMessage(root, eventType), - "response.output_audio_transcript.delta" or - "response.output_audio_transcript.done" or - "response.output_audio.delta" or - "response.output_audio.done" => - CreateOutputTextAudioMessage(root, eventType), - "response.created" or - "response.done" => - CreateResponseCreatedMessage(root, eventType), - "response.output_item.added" or - "response.output_item.done" => - CreateResponseOutItemMessage(root, eventType), - "session.created" or - "session.updated" => - HandleSessionEvent(root), - "mcp_call.in_progress" or - "mcp_call.completed" or - "mcp_call.failed" => - CreateMcpCallMessage(root, eventType), - "mcp_list_tools.in_progress" or - "mcp_list_tools.completed" or - "mcp_list_tools.failed" => - CreateMcpListToolsMessage(root, eventType), - "conversation.item.added" or - "conversation.item.done" => - CreateConversationItemMessage(root, eventType), - _ => new RealtimeServerMessage - { - Type = RealtimeServerMessageType.RawContentOnly, - RawRepresentation = root.Clone(), - }, - }; - - if (serverMessage is not null) - { - _ = _eventChannel.Writer.TryWrite(serverMessage); + responseOptions.Tools.Add(sdkTool); } } - catch (Exception) - { - _ = _eventChannel.Writer.TryWrite(new RealtimeServerMessage - { - Type = RealtimeServerMessageType.RawContentOnly, - RawRepresentation = root.Clone(), - }); - } } - catch (Exception) + + if (responseCreate.MessageId is not null) { - // Failed to parse the raw JSON; nothing to write to the channel. + var cmd = new Sdk.RealtimeClientCommandResponseCreate + { + ResponseOptions = responseOptions, + EventId = responseCreate.MessageId, + }; + await _sessionClient!.SendCommandAsync(cmd, cancellationToken).ConfigureAwait(false); } - } - - #region Helper Methods - - private static void CopyJsonPropertyIfExists(JsonNode? source, JsonObject target, string propertyName) - { - if (source?[propertyName] is JsonNode value) + else { - target[propertyName] = value.DeepClone(); + await _sessionClient!.StartResponseAsync(responseOptions, cancellationToken).ConfigureAwait(false); } } - private static JsonArray CreateModalitiesArray(IEnumerable modalities) - => new([.. modalities.Select(m => JsonValue.Create(m))]); - - private static RealtimeAudioFormat? ParseAudioFormat(JsonElement formatElement) + private async Task SendConversationItemCreateAsync(RealtimeClientConversationItemCreateMessage itemCreate, CancellationToken cancellationToken) { - if (formatElement.ValueKind != JsonValueKind.Object || - !formatElement.TryGetProperty("type", out var typeElement)) + if (itemCreate.Item is null) { - return null; + return; } - string? formatType = typeElement.GetString(); - if (formatType is null) + var sdkItem = ToRealtimeItem(itemCreate.Item); + if (sdkItem is null) { - return null; + return; } - int sampleRate = formatElement.TryGetProperty("rate", out var rateElement) && rateElement.ValueKind == JsonValueKind.Number - ? rateElement.GetInt32() - : 0; - return new RealtimeAudioFormat(formatType, sampleRate); - } - - private static int? ParseMaxOutputTokens(JsonElement element) - { - return element.ValueKind == JsonValueKind.Number - ? element.GetInt32() - : element.ValueKind == JsonValueKind.String && element.GetString() == "inf" - ? int.MaxValue - : null; - } - - private static List? ParseOutputModalities(JsonElement element) - { - if (element.ValueKind != JsonValueKind.Array) + if (itemCreate.MessageId is not null || itemCreate.PreviousId is not null) { - return null; + var cmd = new Sdk.RealtimeClientCommandConversationItemCreate(sdkItem) + { + EventId = itemCreate.MessageId, + PreviousItemId = itemCreate.PreviousId, + }; + await _sessionClient!.SendCommandAsync(cmd, cancellationToken).ConfigureAwait(false); } - - var modalities = new List(); - foreach (var item in element.EnumerateArray()) + else { - if (item.GetString() is string m && !string.IsNullOrEmpty(m)) - { - modalities.Add(m); - } + await _sessionClient!.AddItemAsync(sdkItem, cancellationToken).ConfigureAwait(false); } - - return modalities.Count > 0 ? modalities : null; } - private static JsonObject? SerializeAIFunctionToJson(AIFunction? aiFunction) + private async Task SendInputAudioAppendAsync(RealtimeClientInputAudioBufferAppendMessage audioAppend, CancellationToken cancellationToken) { - if (aiFunction is null || string.IsNullOrEmpty(aiFunction.Name)) + if (audioAppend.Content is null || !audioAppend.Content.HasTopLevelMediaType("audio")) { - return null; + return; } - var toolObj = new JsonObject - { - ["type"] = "function", - ["name"] = aiFunction.Name, - }; + BinaryData audioData = ExtractAudioBinaryData(audioAppend.Content); - if (!string.IsNullOrEmpty(aiFunction.Description)) + if (audioAppend.MessageId is not null) { - toolObj["description"] = aiFunction.Description; + var cmd = new Sdk.RealtimeClientCommandInputAudioBufferAppend(audioData) { EventId = audioAppend.MessageId }; + await _sessionClient!.SendCommandAsync(cmd, cancellationToken).ConfigureAwait(false); + } + else + { + await _sessionClient!.SendInputAudioAsync(audioData, cancellationToken).ConfigureAwait(false); } - - toolObj["parameters"] = JsonNode.Parse(aiFunction.JsonSchema.GetRawText()); - return toolObj; } - private static JsonObject? SerializeMcpToolToJson(HostedMcpServerTool? mcpTool) + private async Task SendRawCommandAsync(RealtimeClientMessage message, CancellationToken cancellationToken) { - if (mcpTool is null) + if (message.RawRepresentation is Sdk.RealtimeClientCommand sdkCmd) { - return null; + await _sessionClient!.SendCommandAsync(sdkCmd, cancellationToken).ConfigureAwait(false); + return; } - var toolObj = new JsonObject + string? jsonString = message.RawRepresentation switch { - ["type"] = "mcp", - ["server_label"] = mcpTool.ServerName, + string s => s, + JsonObject obj => obj.ToJsonString(), + _ => null, }; - if (Uri.TryCreate(mcpTool.ServerAddress, UriKind.Absolute, out _)) + if (jsonString is not null) { - toolObj["server_url"] = mcpTool.ServerAddress; - - if (mcpTool.Headers is { } headers) + // Inject event_id if the message has one but the raw JSON does not. + if (message.MessageId is not null && !jsonString.Contains("\"event_id\"", StringComparison.Ordinal)) { - var headersObj = new JsonObject(); - foreach (var kvp in headers) - { - headersObj[kvp.Key] = kvp.Value; - } - - if (headersObj.Count > 0) - { - toolObj["headers"] = headersObj; - } + jsonString = jsonString.Insert(1, $"\"event_id\":\"{message.MessageId}\","); } - } - else - { - toolObj["connector_id"] = mcpTool.ServerAddress; - if (mcpTool.AuthorizationToken is not null) - { - toolObj["authorization"] = new JsonObject - { - ["token"] = mcpTool.AuthorizationToken, - }; - } + await _sessionClient!.SendCommandAsync(BinaryData.FromString(jsonString), null).ConfigureAwait(false); } + } - if (mcpTool.ServerDescription is not null) + private static Sdk.RealtimeConversationSessionOptions BuildConversationSessionOptions(RealtimeSessionOptions options) + { + var convOptions = new Sdk.RealtimeConversationSessionOptions(); + + // Audio configuration. + var audioOptions = new Sdk.RealtimeConversationSessionAudioOptions(); + var inputAudioOptions = new Sdk.RealtimeConversationSessionInputAudioOptions(); + var outputAudioOptions = new Sdk.RealtimeConversationSessionOutputAudioOptions(); + + if (options.InputAudioFormat is not null) { - toolObj["server_description"] = mcpTool.ServerDescription; + inputAudioOptions.AudioFormat = ToSdkAudioFormat(options.InputAudioFormat); } - if (mcpTool.AllowedTools is { Count: > 0 }) + if (options.NoiseReductionOptions.HasValue) { - toolObj["allowed_tools"] = new JsonArray([.. mcpTool.AllowedTools.Select(t => JsonValue.Create(t))]); + inputAudioOptions.NoiseReduction = new Sdk.RealtimeNoiseReduction( + options.NoiseReductionOptions.Value == NoiseReductionOptions.NearField + ? Sdk.RealtimeNoiseReductionKind.NearField + : Sdk.RealtimeNoiseReductionKind.FarField); } - if (mcpTool.ApprovalMode is not null) + if (options.TranscriptionOptions is not null) { - toolObj["require_approval"] = mcpTool.ApprovalMode switch + inputAudioOptions.AudioTranscriptionOptions = new Sdk.RealtimeAudioTranscriptionOptions { - HostedMcpServerToolAlwaysRequireApprovalMode => (JsonNode)"always", - HostedMcpServerToolNeverRequireApprovalMode => (JsonNode)"never", - HostedMcpServerToolRequireSpecificApprovalMode specific => CreateSpecificApprovalNode(specific), - _ => (JsonNode)"always", + Language = options.TranscriptionOptions.SpeechLanguage, + Model = options.TranscriptionOptions.ModelId, + Prompt = options.TranscriptionOptions.Prompt, }; } - return toolObj; - } - - private static JsonObject CreateSpecificApprovalNode(HostedMcpServerToolRequireSpecificApprovalMode mode) - { - var approvalObj = new JsonObject(); - - if (mode.AlwaysRequireApprovalToolNames is { Count: > 0 }) + if (options.VoiceActivityDetection is ServerVoiceActivityDetection serverVad) { - approvalObj["always"] = new JsonObject + inputAudioOptions.TurnDetection = new Sdk.RealtimeServerVadTurnDetection { - ["tool_names"] = new JsonArray([.. mode.AlwaysRequireApprovalToolNames.Select(t => JsonValue.Create(t))]), + CreateResponseEnabled = serverVad.CreateResponse, + InterruptResponseEnabled = serverVad.InterruptResponse, + DetectionThreshold = (float)serverVad.Threshold, + IdleTimeout = TimeSpan.FromMilliseconds(serverVad.IdleTimeoutInMilliseconds), + PrefixPadding = TimeSpan.FromMilliseconds(serverVad.PrefixPaddingInMilliseconds), + SilenceDuration = TimeSpan.FromMilliseconds(serverVad.SilenceDurationInMilliseconds), }; } - - if (mode.NeverRequireApprovalToolNames is { Count: > 0 }) + else if (options.VoiceActivityDetection is SemanticVoiceActivityDetection semanticVad) { - approvalObj["never"] = new JsonObject + inputAudioOptions.TurnDetection = new Sdk.RealtimeSemanticVadTurnDetection { - ["tool_names"] = new JsonArray([.. mode.NeverRequireApprovalToolNames.Select(t => JsonValue.Create(t))]), + CreateResponseEnabled = semanticVad.CreateResponse, + InterruptResponseEnabled = semanticVad.InterruptResponse, + EagernessLevel = new Sdk.RealtimeSemanticVadEagernessLevel(semanticVad.Eagerness.Value), }; } - - return approvalObj; - } - - private static JsonArray SerializeContentsToJsonArray(IEnumerable contents) - { - var contentsArray = new JsonArray(); - - foreach (var content in contents) + else if (options.VoiceActivityDetection is { } baseVad) { - if (content is TextContent textContent) + // Base VoiceActivityDetection: default to server VAD with basic settings. + inputAudioOptions.TurnDetection = new Sdk.RealtimeServerVadTurnDetection { - contentsArray.Add(new JsonObject - { - ["type"] = "input_text", - ["text"] = textContent.Text, - }); - } - else if (content is DataContent dataContent) - { - if (dataContent.MediaType.StartsWith("audio/", StringComparison.Ordinal)) - { - contentsArray.Add(new JsonObject - { - ["type"] = "input_audio", - ["audio"] = dataContent.Base64Data.ToString(), - }); - } - else if (dataContent.MediaType.StartsWith("image/", StringComparison.Ordinal)) - { - contentsArray.Add(new JsonObject - { - ["type"] = "input_image", - ["image_url"] = dataContent.Uri, - }); - } - } - } - - return contentsArray; - } - - private static ChatRole? ParseChatRole(string? roleString) => roleString switch - { - "assistant" => ChatRole.Assistant, - "user" => ChatRole.User, - "system" => ChatRole.System, - _ => null, - }; - - private static UsageDetails? ParseUsageDetails(JsonElement usageElement, bool requireTypeCheck = false) - { - if (usageElement.ValueKind != JsonValueKind.Object) - { - return null; + CreateResponseEnabled = baseVad.CreateResponse, + InterruptResponseEnabled = baseVad.InterruptResponse, + }; } - if (requireTypeCheck && - (!usageElement.TryGetProperty("type", out var usageTypeElement) || - usageTypeElement.GetString() != "tokens")) + if (options.OutputAudioFormat is not null) { - return null; + outputAudioOptions.AudioFormat = ToSdkAudioFormat(options.OutputAudioFormat); } - var usageData = new UsageDetails(); + outputAudioOptions.Speed = (float)options.VoiceSpeed; - if (usageElement.TryGetProperty("input_tokens", out var inputTokensElement)) + if (options.Voice is not null) { - usageData.InputTokenCount = inputTokensElement.GetInt32(); + outputAudioOptions.Voice = new Sdk.RealtimeVoice(options.Voice); } - if (usageElement.TryGetProperty("output_tokens", out var outputTokensElement)) + audioOptions.InputAudioOptions = inputAudioOptions; + audioOptions.OutputAudioOptions = outputAudioOptions; + convOptions.AudioOptions = audioOptions; + + if (options.Instructions is not null) { - usageData.OutputTokenCount = outputTokensElement.GetInt32(); + convOptions.Instructions = options.Instructions; } - if (usageElement.TryGetProperty("total_tokens", out var totalTokensElement)) + if (options.MaxOutputTokens.HasValue) { - usageData.TotalTokenCount = totalTokensElement.GetInt32(); + convOptions.MaxOutputTokenCount = options.MaxOutputTokens.Value; } - if (usageElement.TryGetProperty("input_token_details", out var inputTokenDetailsElement) && - inputTokenDetailsElement.ValueKind == JsonValueKind.Object) + if (options.Model is not null) { - if (inputTokenDetailsElement.TryGetProperty("audio_tokens", out var audioTokensElement)) - { - usageData.InputAudioTokenCount = audioTokensElement.GetInt32(); - } - - if (inputTokenDetailsElement.TryGetProperty("text_tokens", out var textTokensElement)) - { - usageData.InputTextTokenCount = textTokensElement.GetInt32(); - } + convOptions.Model = options.Model; } - if (usageElement.TryGetProperty("output_token_details", out var outputTokenDetailsElement) && - outputTokenDetailsElement.ValueKind == JsonValueKind.Object) + if (options.OutputModalities is not null) { - if (outputTokenDetailsElement.TryGetProperty("audio_tokens", out var audioTokensElement)) - { - usageData.OutputAudioTokenCount = audioTokensElement.GetInt32(); - } - - if (outputTokenDetailsElement.TryGetProperty("text_tokens", out var textTokensElement)) + foreach (var modality in options.OutputModalities) { - usageData.OutputTextTokenCount = textTokensElement.GetInt32(); + convOptions.OutputModalities.Add(new Sdk.RealtimeOutputModality(modality)); } } - return usageData; - } - - private static List ParseContentItems(JsonElement contentElements) - { - if (contentElements.ValueKind != JsonValueKind.Array) + if (options.ToolMode is { } toolMode) { - return []; + convOptions.ToolChoice = ToSdkToolChoice(toolMode); } - var contentList = new List(); - - foreach (var contentElement in contentElements.EnumerateArray()) + if (options.Tools is not null) { - if (!contentElement.TryGetProperty("type", out var contentTypeElement)) - { - continue; - } - - string? contentType = contentTypeElement.GetString(); - - if (contentType == "input_text") + foreach (var tool in options.Tools) { - if (contentElement.TryGetProperty("text", out var textElement)) - { - contentList.Add(new TextContent(textElement.GetString())); - } - else if (contentElement.TryGetProperty("transcript", out var transcriptElement)) + if (ToRealtimeTool(tool) is Sdk.RealtimeTool sdkTool) { - contentList.Add(new TextContent(transcriptElement.GetString())); + convOptions.Tools.Add(sdkTool); } } - else if (contentType == "input_audio" && contentElement.TryGetProperty("audio", out var audioDataElement)) - { - contentList.Add(new DataContent($"data:audio/pcm;base64,{audioDataElement.GetString()}")); - } - else if (contentType == "input_image" && contentElement.TryGetProperty("image_url", out var imageUrlElement)) - { - contentList.Add(new DataContent(imageUrlElement.GetString()!)); - } } - return contentList; + return convOptions; } - private static RealtimeContentItem? ParseRealtimeContentItem(JsonElement itemElement) + private static Sdk.RealtimeTranscriptionSessionOptions BuildTranscriptionSessionOptions(RealtimeSessionOptions options) { - if (!itemElement.TryGetProperty("type", out var itemTypeElement)) - { - return null; - } + var transOptions = new Sdk.RealtimeTranscriptionSessionOptions(); - string? id = itemElement.TryGetProperty("id", out var idElement) ? idElement.GetString() : null; - string? itemType = itemTypeElement.GetString(); - - if (itemType == "message") + if (options.InputAudioFormat is not null || options.TranscriptionOptions is not null || + options.VoiceActivityDetection is not null || options.NoiseReductionOptions.HasValue) { - if (!itemElement.TryGetProperty("content", out var contentElements) || - contentElements.ValueKind != JsonValueKind.Array) - { - return null; - } - - ChatRole? chatRole = itemElement.TryGetProperty("role", out var roleElement) - ? ParseChatRole(roleElement.GetString()) - : null; - - return new RealtimeContentItem(ParseContentItems(contentElements), id, chatRole); - } - else if (itemType == "function_call" && - itemElement.TryGetProperty("name", out var nameElement) && - itemElement.TryGetProperty("call_id", out var callerIdElement)) - { - IDictionary? arguments = null; + var inputAudioOptions = new Sdk.RealtimeTranscriptionSessionInputAudioOptions(); - if (itemElement.TryGetProperty("arguments", out var argumentsElement)) + if (options.InputAudioFormat is not null) { - if (argumentsElement.ValueKind == JsonValueKind.String) - { - string argumentsJson = argumentsElement.GetString()!; - arguments = string.IsNullOrEmpty(argumentsJson) ? null : JsonSerializer.Deserialize>(argumentsJson); - } - else if (argumentsElement.ValueKind == JsonValueKind.Object) - { - arguments = new AdditionalPropertiesDictionary(); - foreach (var argProperty in argumentsElement.EnumerateObject()) - { - arguments[argProperty.Name] = argProperty.Value.ValueKind == JsonValueKind.String - ? argProperty.Value.GetString() - : JsonSerializer.Deserialize(argProperty.Value.GetRawText()); - } - } + inputAudioOptions.AudioFormat = ToSdkAudioFormat(options.InputAudioFormat); } - return new RealtimeContentItem( - [new FunctionCallContent(callerIdElement.GetString()!, nameElement.GetString()!, arguments)], - id); - } - else if (itemType == "mcp_call" && - itemElement.TryGetProperty("name", out var mcpNameElement)) - { - string? serverLabel = itemElement.TryGetProperty("server_label", out var serverLabelElement) ? serverLabelElement.GetString() : null; - string callId = id ?? string.Empty; - - IReadOnlyDictionary? mcpArguments = null; - if (itemElement.TryGetProperty("arguments", out var mcpArgumentsElement) && mcpArgumentsElement.ValueKind == JsonValueKind.String) + if (options.TranscriptionOptions is not null) { - string argsJson = mcpArgumentsElement.GetString()!; - if (!string.IsNullOrEmpty(argsJson)) + inputAudioOptions.AudioTranscriptionOptions = new Sdk.RealtimeAudioTranscriptionOptions { - mcpArguments = JsonSerializer.Deserialize>(argsJson); - } + Language = options.TranscriptionOptions.SpeechLanguage, + Model = options.TranscriptionOptions.ModelId, + Prompt = options.TranscriptionOptions.Prompt, + }; } - var contents = new List(); - contents.Add(new McpServerToolCallContent(callId, mcpNameElement.GetString()!, serverLabel) - { - Arguments = mcpArguments, - }); - - // Parse output/error into McpServerToolResultContent (same pattern as OpenAIResponsesChatClient.AddMcpToolCallContent) - if (itemElement.TryGetProperty("output", out var outputElement) || - itemElement.TryGetProperty("error", out _)) + if (options.NoiseReductionOptions.HasValue) { - AIContent resultContent = itemElement.TryGetProperty("error", out var errorElement) && errorElement.ValueKind != JsonValueKind.Null - ? new ErrorContent(errorElement.ToString()) - : new TextContent(outputElement.ValueKind == JsonValueKind.String ? outputElement.GetString() : null); - - contents.Add(new McpServerToolResultContent(callId) - { - Output = [resultContent], - RawRepresentation = itemElement.Clone(), - }); + inputAudioOptions.NoiseReduction = new Sdk.RealtimeNoiseReduction( + options.NoiseReductionOptions.Value == NoiseReductionOptions.NearField + ? Sdk.RealtimeNoiseReductionKind.NearField + : Sdk.RealtimeNoiseReductionKind.FarField); } - return new RealtimeContentItem(contents, id); - } - else if (itemType == "mcp_approval_request" && - itemElement.TryGetProperty("name", out var approvalNameElement)) - { - string approvalId = id ?? string.Empty; - string? approvalServerLabel = itemElement.TryGetProperty("server_label", out var approvalServerElement) ? approvalServerElement.GetString() : null; - - IReadOnlyDictionary? approvalArguments = null; - if (itemElement.TryGetProperty("arguments", out var approvalArgsElement) && approvalArgsElement.ValueKind == JsonValueKind.String) + if (options.VoiceActivityDetection is ServerVoiceActivityDetection serverVad) { - string argsJson = approvalArgsElement.GetString()!; - if (!string.IsNullOrEmpty(argsJson)) + inputAudioOptions.TurnDetection = new Sdk.RealtimeServerVadTurnDetection { - approvalArguments = JsonSerializer.Deserialize>(argsJson); - } + CreateResponseEnabled = serverVad.CreateResponse, + InterruptResponseEnabled = serverVad.InterruptResponse, + DetectionThreshold = (float)serverVad.Threshold, + IdleTimeout = TimeSpan.FromMilliseconds(serverVad.IdleTimeoutInMilliseconds), + PrefixPadding = TimeSpan.FromMilliseconds(serverVad.PrefixPaddingInMilliseconds), + SilenceDuration = TimeSpan.FromMilliseconds(serverVad.SilenceDurationInMilliseconds), + }; } - - var toolCall = new McpServerToolCallContent(approvalId, approvalNameElement.GetString()!, approvalServerLabel) - { - Arguments = approvalArguments, - RawRepresentation = itemElement.Clone(), - }; - - return new RealtimeContentItem( - [new McpServerToolApprovalRequestContent(approvalId, toolCall) { RawRepresentation = itemElement.Clone() }], - id); - } - - return null; - } - - #endregion - - #region Message Creation Methods - - private RealtimeServerMessage? HandleSessionEvent(JsonElement root) - { - if (root.TryGetProperty("session", out var sessionElement)) - { - Options = DeserializeSessionOptions(sessionElement, Options); - } - - return new RealtimeServerMessage - { - Type = RealtimeServerMessageType.RawContentOnly, - RawRepresentation = root.Clone(), - }; - } - - private static RealtimeSessionOptions DeserializeSessionOptions(JsonElement session, RealtimeSessionOptions? previousOptions) - { - RealtimeSessionKind sessionKind = RealtimeSessionKind.Realtime; - if (session.TryGetProperty("type", out var typeElement) && typeElement.GetString() == "transcription") - { - sessionKind = RealtimeSessionKind.Transcription; - } - - string? model = session.TryGetProperty("model", out var modelElement) ? modelElement.GetString() : null; - - string? instructions = session.TryGetProperty("instructions", out var instructionsElement) && instructionsElement.ValueKind == JsonValueKind.String - ? instructionsElement.GetString() - : null; - - int? maxOutputTokens = session.TryGetProperty("max_output_tokens", out var maxTokensElement) - ? ParseMaxOutputTokens(maxTokensElement) - : null; - - IReadOnlyList? outputModalities = session.TryGetProperty("output_modalities", out var modalitiesElement) - ? ParseOutputModalities(modalitiesElement) - : null; - - RealtimeAudioFormat? inputAudioFormat = null; - NoiseReductionOptions? noiseReductionOptions = null; - TranscriptionOptions? transcriptionOptions = null; - VoiceActivityDetection? voiceActivityDetection = null; - RealtimeAudioFormat? outputAudioFormat = null; - double voiceSpeed = 1.0; - string? voice = null; - - // Audio configuration. - if (session.TryGetProperty("audio", out var audioElement) && - audioElement.ValueKind == JsonValueKind.Object) - { - // Input audio. - if (audioElement.TryGetProperty("input", out var inputElement) && - inputElement.ValueKind == JsonValueKind.Object) + else if (options.VoiceActivityDetection is SemanticVoiceActivityDetection semanticVad) { - if (inputElement.TryGetProperty("format", out var inputFormatElement)) + inputAudioOptions.TurnDetection = new Sdk.RealtimeSemanticVadTurnDetection { - inputAudioFormat = ParseAudioFormat(inputFormatElement); - } - - if (inputElement.TryGetProperty("noise_reduction", out var noiseElement) && - noiseElement.ValueKind == JsonValueKind.Object && - noiseElement.TryGetProperty("type", out var noiseTypeElement)) - { - noiseReductionOptions = noiseTypeElement.GetString() switch - { - "near_field" => NoiseReductionOptions.NearField, - "far_field" => NoiseReductionOptions.FarField, - _ => null, - }; - } - - if (inputElement.TryGetProperty("transcription", out var transcriptionElement) && - transcriptionElement.ValueKind == JsonValueKind.Object) - { - string? language = transcriptionElement.TryGetProperty("language", out var langElement) ? langElement.GetString() : null; - string? transcriptionModel = transcriptionElement.TryGetProperty("model", out var modelEl) ? modelEl.GetString() : null; - string? prompt = transcriptionElement.TryGetProperty("prompt", out var promptElement) ? promptElement.GetString() : null; - - if (language is not null && transcriptionModel is not null) - { - transcriptionOptions = new TranscriptionOptions { SpeechLanguage = language, ModelId = transcriptionModel, Prompt = prompt }; - } - } - - // Turn detection (VAD). - if (inputElement.TryGetProperty("turn_detection", out var turnDetectionElement) && - turnDetectionElement.ValueKind == JsonValueKind.Object && - turnDetectionElement.TryGetProperty("type", out var vadTypeElement)) - { - voiceActivityDetection = ParseVoiceActivityDetection(vadTypeElement.GetString(), turnDetectionElement); - } + CreateResponseEnabled = semanticVad.CreateResponse, + InterruptResponseEnabled = semanticVad.InterruptResponse, + EagernessLevel = new Sdk.RealtimeSemanticVadEagernessLevel(semanticVad.Eagerness.Value), + }; } - - // Output audio. - if (audioElement.TryGetProperty("output", out var outputElement) && - outputElement.ValueKind == JsonValueKind.Object) + else if (options.VoiceActivityDetection is { } baseVad) { - if (outputElement.TryGetProperty("format", out var outputFormatElement)) + inputAudioOptions.TurnDetection = new Sdk.RealtimeServerVadTurnDetection { - outputAudioFormat = ParseAudioFormat(outputFormatElement); - } - - if (outputElement.TryGetProperty("speed", out var speedElement) && speedElement.ValueKind == JsonValueKind.Number) - { - voiceSpeed = speedElement.GetDouble(); - } - - if (outputElement.TryGetProperty("voice", out var voiceElement)) - { - if (voiceElement.ValueKind == JsonValueKind.String) - { - voice = voiceElement.GetString(); - } - else if (voiceElement.ValueKind == JsonValueKind.Object && - voiceElement.TryGetProperty("id", out var voiceIdElement)) - { - voice = voiceIdElement.GetString(); - } - } + CreateResponseEnabled = baseVad.CreateResponse, + InterruptResponseEnabled = baseVad.InterruptResponse, + }; } - } - - return new RealtimeSessionOptions - { - SessionKind = sessionKind, - Model = model, - Instructions = instructions, - MaxOutputTokens = maxOutputTokens, - OutputModalities = outputModalities, - InputAudioFormat = inputAudioFormat, - NoiseReductionOptions = noiseReductionOptions, - TranscriptionOptions = transcriptionOptions, - VoiceActivityDetection = voiceActivityDetection, - OutputAudioFormat = outputAudioFormat, - VoiceSpeed = voiceSpeed, - Voice = voice, - // Preserve client-side properties that the server cannot round-trip - // as typed objects (tools are returned as JSON schemas, not AITool instances). - Tools = previousOptions?.Tools, - ToolMode = previousOptions?.ToolMode, - }; + transOptions.AudioOptions = new Sdk.RealtimeTranscriptionSessionAudioOptions + { + InputAudioOptions = inputAudioOptions, + }; + } + + return transOptions; } - private static VoiceActivityDetection? ParseVoiceActivityDetection(string? vadType, JsonElement turnDetectionElement) + private static Sdk.RealtimeTool? ToRealtimeTool(AITool tool) { - if (vadType == "server_vad") + if (tool is AIFunction aiFunction && !string.IsNullOrEmpty(aiFunction.Name)) { - return new ServerVoiceActivityDetection - { - CreateResponse = turnDetectionElement.TryGetProperty("create_response", out var crElement) && crElement.GetBoolean(), - InterruptResponse = turnDetectionElement.TryGetProperty("interrupt_response", out var irElement) && irElement.GetBoolean(), - IdleTimeoutInMilliseconds = turnDetectionElement.TryGetProperty("idle_timeout_ms", out var itElement) && itElement.ValueKind == JsonValueKind.Number ? itElement.GetInt32() : 0, - PrefixPaddingInMilliseconds = turnDetectionElement.TryGetProperty("prefix_padding_ms", out var ppElement) && ppElement.ValueKind == JsonValueKind.Number ? ppElement.GetInt32() : 300, - SilenceDurationInMilliseconds = turnDetectionElement.TryGetProperty("silence_duration_ms", out var sdElement) && sdElement.ValueKind == JsonValueKind.Number ? sdElement.GetInt32() : 500, - Threshold = turnDetectionElement.TryGetProperty("threshold", out var thElement) && thElement.ValueKind == JsonValueKind.Number ? thElement.GetDouble() : 0.5, - }; + return OpenAIRealtimeConversationClient.ToOpenAIRealtimeFunctionTool(aiFunction); } - if (vadType == "semantic_vad") + if (tool is HostedMcpServerTool mcpTool) { - return new SemanticVoiceActivityDetection - { - CreateResponse = turnDetectionElement.TryGetProperty("create_response", out var crElement) && crElement.GetBoolean(), - InterruptResponse = turnDetectionElement.TryGetProperty("interrupt_response", out var irElement) && irElement.GetBoolean(), - Eagerness = turnDetectionElement.TryGetProperty("eagerness", out var eagernessElement) && eagernessElement.GetString() is string eagerness - ? new SemanticEagerness(eagerness) - : SemanticEagerness.Auto, - }; + return ToRealtimeMcpTool(mcpTool); } return null; } - private static RealtimeServerErrorMessage? CreateErrorMessage(JsonElement root) + private static Sdk.RealtimeMcpTool ToRealtimeMcpTool(HostedMcpServerTool mcpTool) { - if (!root.TryGetProperty("error", out var errorElement) || - !errorElement.TryGetProperty("message", out var messageElement)) + Sdk.RealtimeMcpTool sdkTool; + + if (Uri.TryCreate(mcpTool.ServerAddress, UriKind.Absolute, out var uri)) { - return null; - } + sdkTool = new Sdk.RealtimeMcpTool(mcpTool.ServerName, uri); - var msg = new RealtimeServerErrorMessage + if (mcpTool.Headers is { } headers) + { + var sdkHeaders = new Dictionary(); + foreach (var kvp in headers) + { + sdkHeaders[kvp.Key] = kvp.Value; + } + + sdkTool.Headers = sdkHeaders; + } + } + else { - Error = new ErrorContent(messageElement.GetString()), - }; + sdkTool = new Sdk.RealtimeMcpTool(mcpTool.ServerName, new Sdk.RealtimeMcpToolConnectorId(mcpTool.ServerAddress)); + + if (mcpTool.AuthorizationToken is not null) + { + sdkTool.AuthorizationToken = mcpTool.AuthorizationToken; + } + } - if (errorElement.TryGetProperty("code", out var codeElement)) + if (mcpTool.ServerDescription is not null) { - msg.Error.ErrorCode = codeElement.GetString(); + sdkTool.ServerDescription = mcpTool.ServerDescription; } - if (root.TryGetProperty("event_id", out var eventIdElement)) + if (mcpTool.AllowedTools is { Count: > 0 }) { - msg.MessageId = eventIdElement.GetString(); + sdkTool.AllowedTools = new Sdk.RealtimeMcpToolFilter(); + foreach (var toolName in mcpTool.AllowedTools) + { + sdkTool.AllowedTools.ToolNames.Add(toolName); + } } - if (errorElement.TryGetProperty("param", out var paramElement)) + if (mcpTool.ApprovalMode is not null) { - msg.Error.Details = paramElement.GetString(); + sdkTool.ToolCallApprovalPolicy = mcpTool.ApprovalMode switch + { + HostedMcpServerToolAlwaysRequireApprovalMode => Sdk.RealtimeDefaultMcpToolCallApprovalPolicy.AlwaysRequireApproval, + HostedMcpServerToolNeverRequireApprovalMode => Sdk.RealtimeDefaultMcpToolCallApprovalPolicy.NeverRequireApproval, + HostedMcpServerToolRequireSpecificApprovalMode specific => ToSdkCustomApprovalPolicy(specific), + _ => Sdk.RealtimeDefaultMcpToolCallApprovalPolicy.AlwaysRequireApproval, + }; } - return msg; + return sdkTool; } - private static RealtimeServerInputAudioTranscriptionMessage? CreateInputAudioTranscriptionMessage(JsonElement root, string messageType) + private static Sdk.RealtimeMcpToolCallApprovalPolicy ToSdkCustomApprovalPolicy(HostedMcpServerToolRequireSpecificApprovalMode mode) { - RealtimeServerMessageType serverMessageType = messageType switch - { - "conversation.item.input_audio_transcription.delta" => RealtimeServerMessageType.InputAudioTranscriptionDelta, - "conversation.item.input_audio_transcription.completed" => RealtimeServerMessageType.InputAudioTranscriptionCompleted, - "conversation.item.input_audio_transcription.failed" => RealtimeServerMessageType.InputAudioTranscriptionFailed, - _ => throw new InvalidOperationException($"Unknown message type: {messageType}"), - }; + var custom = new Sdk.RealtimeCustomMcpToolCallApprovalPolicy(); - var msg = new RealtimeServerInputAudioTranscriptionMessage(serverMessageType); - - if (root.TryGetProperty("event_id", out var eventIdElement)) + if (mode.AlwaysRequireApprovalToolNames is { Count: > 0 }) { - msg.MessageId = eventIdElement.GetString(); + custom.ToolsAlwaysRequiringApproval = new Sdk.RealtimeMcpToolFilter(); + foreach (var name in mode.AlwaysRequireApprovalToolNames) + { + custom.ToolsAlwaysRequiringApproval.ToolNames.Add(name); + } } - if (root.TryGetProperty("content_index", out var contentIndexElement)) + if (mode.NeverRequireApprovalToolNames is { Count: > 0 }) { - msg.ContentIndex = contentIndexElement.GetInt32(); + custom.ToolsNeverRequiringApproval = new Sdk.RealtimeMcpToolFilter(); + foreach (var name in mode.NeverRequireApprovalToolNames) + { + custom.ToolsNeverRequiringApproval.ToolNames.Add(name); + } } - if (root.TryGetProperty("item_id", out var itemIdElement)) + return custom; + } + + private static Sdk.RealtimeItem? ToRealtimeItem(RealtimeContentItem? contentItem) + { + if (contentItem?.Contents is null or { Count: 0 }) { - msg.ItemId = itemIdElement.GetString(); + return null; } - if (root.TryGetProperty("delta", out var deltaElement)) + var firstContent = contentItem.Contents[0]; + + if (firstContent is FunctionResultContent functionResult) { - msg.Transcription = deltaElement.GetString(); + return Sdk.RealtimeItem.CreateFunctionCallOutputItem( + functionResult.CallId ?? string.Empty, + functionResult.Result is not null ? JsonSerializer.Serialize(functionResult.Result) : string.Empty); } - if (msg.Transcription is null && root.TryGetProperty("transcript", out deltaElement)) + if (firstContent is FunctionCallContent functionCall) { - msg.Transcription = deltaElement.GetString(); + var arguments = functionCall.Arguments is not null + ? BinaryData.FromString(JsonSerializer.Serialize(functionCall.Arguments)) + : BinaryData.FromString("{}"); + return Sdk.RealtimeItem.CreateFunctionCallItem( + functionCall.CallId ?? string.Empty, + functionCall.Name, + arguments); } - if (root.TryGetProperty("error", out var errorElement) && - errorElement.TryGetProperty("message", out var errorMsgElement)) + if (firstContent is McpServerToolApprovalResponseContent approvalResponse) { - var errorContent = new ErrorContent(errorMsgElement.GetString()); + return Sdk.RealtimeItem.CreateMcpApprovalResponseItem( + approvalResponse.Id ?? string.Empty, + approvalResponse.Approved); + } - if (errorElement.TryGetProperty("code", out var errorCodeElement)) + // Message item with content parts. + var contentParts = new List(); + foreach (var content in contentItem.Contents) + { + if (content is TextContent textContent) { - errorContent.ErrorCode = errorCodeElement.GetString(); + contentParts.Add(new Sdk.RealtimeInputTextMessageContentPart(textContent.Text ?? string.Empty)); } - - if (errorElement.TryGetProperty("param", out var errorParamElement)) + else if (content is DataContent dataContent) { - errorContent.Details = errorParamElement.GetString(); + if (dataContent.MediaType?.StartsWith("audio/", StringComparison.Ordinal) == true) + { + contentParts.Add(new Sdk.RealtimeInputAudioMessageContentPart( + BinaryData.FromBytes(dataContent.Data.ToArray()))); + } + else if (dataContent.MediaType?.StartsWith("image/", StringComparison.Ordinal) == true && dataContent.Uri is not null) + { + contentParts.Add(new Sdk.RealtimeInputImageMessageContentPart(new Uri(dataContent.Uri))); + } } - - msg.Error = errorContent; } - if (root.TryGetProperty("usage", out var usageElement)) + if (contentParts.Count == 0) { - msg.Usage = ParseUsageDetails(usageElement, requireTypeCheck: true); + return null; } - return msg; - } - - private static RealtimeServerOutputTextAudioMessage? CreateOutputTextAudioMessage(JsonElement root, string messageType) - { - RealtimeServerMessageType serverMessageType = messageType switch + var role = contentItem.Role?.Value switch { - "response.output_audio.delta" => RealtimeServerMessageType.OutputAudioDelta, - "response.output_audio.done" => RealtimeServerMessageType.OutputAudioDone, - "response.output_audio_transcript.delta" => RealtimeServerMessageType.OutputAudioTranscriptionDelta, - "response.output_audio_transcript.done" => RealtimeServerMessageType.OutputAudioTranscriptionDone, - _ => throw new InvalidOperationException($"Unknown message type: {messageType}"), + "assistant" => Sdk.RealtimeMessageRole.Assistant, + "system" => Sdk.RealtimeMessageRole.System, + _ => Sdk.RealtimeMessageRole.User, }; - var msg = new RealtimeServerOutputTextAudioMessage(serverMessageType); - - if (root.TryGetProperty("event_id", out var eventIdElement)) + var messageItem = new Sdk.RealtimeMessageItem(role, contentParts); + if (contentItem.Id is not null) { - msg.MessageId = eventIdElement.GetString(); + messageItem.Id = contentItem.Id; } - if (root.TryGetProperty("response_id", out var responseIdElement)) - { - msg.ResponseId = responseIdElement.GetString(); - } + return messageItem; + } - if (root.TryGetProperty("item_id", out var itemIdElement)) - { - msg.ItemId = itemIdElement.GetString(); - } + private static Sdk.RealtimeToolChoice ToSdkToolChoice(ChatToolMode toolMode) => toolMode switch + { + RequiredChatToolMode r when r.RequiredFunctionName is not null => + new Sdk.RealtimeToolChoice(new Sdk.RealtimeCustomFunctionToolChoice(r.RequiredFunctionName)), + RequiredChatToolMode => Sdk.RealtimeDefaultToolChoice.Required, + NoneChatToolMode => Sdk.RealtimeDefaultToolChoice.None, + _ => Sdk.RealtimeDefaultToolChoice.Auto, + }; - if (root.TryGetProperty("output_index", out var outputIndexElement)) + private static Sdk.RealtimeAudioFormat? ToSdkAudioFormat(RealtimeAudioFormat? format) + { + if (format is null) { - msg.OutputIndex = outputIndexElement.GetInt32(); + return null; } - if (root.TryGetProperty("content_index", out var contentIndexElement)) + return format.MediaType switch { - msg.ContentIndex = contentIndexElement.GetInt32(); - } + "audio/pcm" => new Sdk.RealtimePcmAudioFormat(), + "audio/pcmu" => new Sdk.RealtimePcmuAudioFormat(), + "audio/pcma" => new Sdk.RealtimePcmaAudioFormat(), + _ => null, + }; + } - if (root.TryGetProperty("delta", out var deltaElement)) - { - if (serverMessageType == RealtimeServerMessageType.OutputAudioDelta) - { - msg.Audio = deltaElement.GetString(); - } - else - { - msg.Text = deltaElement.GetString(); - } - } + private static BinaryData ExtractAudioBinaryData(DataContent content) + { + string dataUri = content.Uri?.ToString() ?? string.Empty; + int commaIndex = dataUri.LastIndexOf(','); - if (msg.Text is null && root.TryGetProperty("transcript", out deltaElement)) + if (commaIndex >= 0 && commaIndex < dataUri.Length - 1) { - msg.Text = deltaElement.GetString(); + string base64 = dataUri.Substring(commaIndex + 1); + return BinaryData.FromBytes(Convert.FromBase64String(base64)); } - return msg; + return BinaryData.FromBytes(content.Data.ToArray()); } - private static RealtimeServerResponseOutputItemMessage? CreateResponseOutItemMessage(JsonElement root, string messageType) - { - RealtimeServerMessageType serverMessageType = messageType switch - { - "response.output_item.added" => RealtimeServerMessageType.ResponseOutputItemAdded, - "response.output_item.done" => RealtimeServerMessageType.ResponseOutputItemDone, - _ => throw new InvalidOperationException($"Unknown message type: {messageType}"), - }; + #endregion - var msg = new RealtimeServerResponseOutputItemMessage(serverMessageType); + #region Receive Helpers (SDK → MEAI) - if (root.TryGetProperty("event_id", out var eventIdElement)) + private RealtimeServerMessage? MapServerUpdate(Sdk.RealtimeServerUpdate update) => update switch + { + Sdk.RealtimeServerUpdateError e => MapError(e), + Sdk.RealtimeServerUpdateSessionCreated e => HandleSessionEvent(e.Session, e), + Sdk.RealtimeServerUpdateSessionUpdated e => HandleSessionEvent(e.Session, e), + Sdk.RealtimeServerUpdateResponseCreated e => MapResponseCreatedOrDone(e.EventId, e.Response, RealtimeServerMessageType.ResponseCreated, e), + Sdk.RealtimeServerUpdateResponseDone e => MapResponseCreatedOrDone(e.EventId, e.Response, RealtimeServerMessageType.ResponseDone, e), + Sdk.RealtimeServerUpdateResponseOutputItemAdded e => MapResponseOutputItem(e.EventId, e.ResponseId, e.OutputIndex, e.Item, RealtimeServerMessageType.ResponseOutputItemAdded, e), + Sdk.RealtimeServerUpdateResponseOutputItemDone e => MapResponseOutputItem(e.EventId, e.ResponseId, e.OutputIndex, e.Item, RealtimeServerMessageType.ResponseOutputItemDone, e), + Sdk.RealtimeServerUpdateResponseOutputAudioDelta e => new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputAudioDelta) + { + MessageId = e.EventId, + ResponseId = e.ResponseId, + ItemId = e.ItemId, + OutputIndex = e.OutputIndex, + ContentIndex = e.ContentIndex, + Audio = e.Delta is not null ? Convert.ToBase64String(e.Delta.ToArray()) : null, + RawRepresentation = e, + }, + Sdk.RealtimeServerUpdateResponseOutputAudioDone e => new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputAudioDone) + { + MessageId = e.EventId, + ResponseId = e.ResponseId, + ItemId = e.ItemId, + OutputIndex = e.OutputIndex, + ContentIndex = e.ContentIndex, + RawRepresentation = e, + }, + Sdk.RealtimeServerUpdateResponseOutputAudioTranscriptDelta e => new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputAudioTranscriptionDelta) + { + MessageId = e.EventId, + ResponseId = e.ResponseId, + ItemId = e.ItemId, + OutputIndex = e.OutputIndex, + ContentIndex = e.ContentIndex, + Text = e.Delta, + RawRepresentation = e, + }, + Sdk.RealtimeServerUpdateResponseOutputAudioTranscriptDone e => new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputAudioTranscriptionDone) + { + MessageId = e.EventId, + ResponseId = e.ResponseId, + ItemId = e.ItemId, + OutputIndex = e.OutputIndex, + ContentIndex = e.ContentIndex, + Text = e.Transcript, + RawRepresentation = e, + }, + Sdk.RealtimeServerUpdateConversationItemInputAudioTranscriptionDelta e => MapInputTranscriptionDelta(e), + Sdk.RealtimeServerUpdateConversationItemInputAudioTranscriptionCompleted e => MapInputTranscriptionCompleted(e), + Sdk.RealtimeServerUpdateConversationItemInputAudioTranscriptionFailed e => MapInputTranscriptionFailed(e), + Sdk.RealtimeServerUpdateConversationItemAdded e => MapConversationItem(e.EventId, e.Item, RealtimeServerMessageType.ResponseOutputItemAdded, e), + Sdk.RealtimeServerUpdateConversationItemDone e => MapConversationItem(e.EventId, e.Item, RealtimeServerMessageType.ResponseOutputItemDone, e), + Sdk.RealtimeServerUpdateResponseMcpCallInProgress e => MapMcpCallEvent(e.EventId, e.ItemId, e.OutputIndex, RealtimeServerMessageType.McpCallInProgress, e), + Sdk.RealtimeServerUpdateResponseMcpCallCompleted e => MapMcpCallEvent(e.EventId, e.ItemId, e.OutputIndex, RealtimeServerMessageType.McpCallCompleted, e), + Sdk.RealtimeServerUpdateResponseMcpCallFailed e => MapMcpCallEvent(e.EventId, e.ItemId, e.OutputIndex, RealtimeServerMessageType.McpCallFailed, e), + Sdk.RealtimeServerUpdateMcpListToolsInProgress e => MapMcpListToolsEvent(e.EventId, e.ItemId, RealtimeServerMessageType.McpListToolsInProgress, e), + Sdk.RealtimeServerUpdateMcpListToolsCompleted e => MapMcpListToolsEvent(e.EventId, e.ItemId, RealtimeServerMessageType.McpListToolsCompleted, e), + Sdk.RealtimeServerUpdateMcpListToolsFailed e => MapMcpListToolsEvent(e.EventId, e.ItemId, RealtimeServerMessageType.McpListToolsFailed, e), + _ => new RealtimeServerMessage { - msg.MessageId = eventIdElement.GetString(); - } + Type = RealtimeServerMessageType.RawContentOnly, + RawRepresentation = update, + }, + }; - if (root.TryGetProperty("response_id", out var responseIdElement)) + private static RealtimeServerErrorMessage MapError(Sdk.RealtimeServerUpdateError e) + { + var msg = new RealtimeServerErrorMessage { - msg.ResponseId = responseIdElement.GetString(); - } + MessageId = e.EventId, + Error = new ErrorContent(e.Error?.Message), + RawRepresentation = e, + }; - if (root.TryGetProperty("output_index", out var outputIndexElement)) + if (e.Error?.Code is not null) { - msg.OutputIndex = outputIndexElement.GetInt32(); + msg.Error.ErrorCode = e.Error.Code; } - if (root.TryGetProperty("item", out var itemElement)) + if (e.Error?.ParameterName is not null) { - msg.Item = ParseRealtimeContentItem(itemElement); + msg.Error.Details = e.Error.ParameterName; } return msg; } - private static RealtimeServerResponseCreatedMessage? CreateResponseCreatedMessage(JsonElement root, string messageType) + private RealtimeServerMessage HandleSessionEvent(Sdk.RealtimeSession? session, Sdk.RealtimeServerUpdate update) { - RealtimeServerMessageType serverMessageType = messageType switch - { - "response.created" => RealtimeServerMessageType.ResponseCreated, - "response.done" => RealtimeServerMessageType.ResponseDone, - _ => throw new InvalidOperationException($"Unknown message type: {messageType}"), - }; - - if (!root.TryGetProperty("response", out var responseElement)) + if (session is Sdk.RealtimeConversationSession convSession) { - return null; + Options = MapConversationSessionToOptions(convSession); } - var msg = new RealtimeServerResponseCreatedMessage(serverMessageType); - - if (root.TryGetProperty("event_id", out var eventIdElement)) + return new RealtimeServerMessage { - msg.MessageId = eventIdElement.GetString(); - } + Type = RealtimeServerMessageType.RawContentOnly, + RawRepresentation = update, + }; + } + + private RealtimeSessionOptions MapConversationSessionToOptions(Sdk.RealtimeConversationSession session) + { + RealtimeAudioFormat? inputAudioFormat = null; + NoiseReductionOptions? noiseReduction = null; + TranscriptionOptions? transcription = null; + VoiceActivityDetection? vad = null; + RealtimeAudioFormat? outputAudioFormat = null; + double voiceSpeed = 1.0; + string? voice = null; - if (responseElement.TryGetProperty("audio", out var responseAudioElement) && - responseAudioElement.ValueKind == JsonValueKind.Object && - responseAudioElement.TryGetProperty("output", out var outputElement) && - outputElement.ValueKind == JsonValueKind.Object) + if (session.AudioOptions is { } audioOptions) { - if (outputElement.TryGetProperty("format", out var formatElement)) + if (audioOptions.InputAudioOptions is { } inputOpts) { - msg.OutputAudioOptions = ParseAudioFormat(formatElement); + inputAudioFormat = MapSdkAudioFormat(inputOpts.AudioFormat); + + if (inputOpts.NoiseReduction is { } nr) + { + noiseReduction = nr.Kind == Sdk.RealtimeNoiseReductionKind.NearField + ? NoiseReductionOptions.NearField + : NoiseReductionOptions.FarField; + } + + if (inputOpts.AudioTranscriptionOptions is { } transcriptionOpts) + { + transcription = new TranscriptionOptions + { + SpeechLanguage = transcriptionOpts.Language, + ModelId = transcriptionOpts.Model, + Prompt = transcriptionOpts.Prompt, + }; + } + + if (inputOpts.TurnDetection is Sdk.RealtimeServerVadTurnDetection serverVad) + { + vad = new ServerVoiceActivityDetection + { + CreateResponse = serverVad.CreateResponseEnabled ?? false, + InterruptResponse = serverVad.InterruptResponseEnabled ?? false, + Threshold = serverVad.DetectionThreshold ?? 0.5, + IdleTimeoutInMilliseconds = (int)(serverVad.IdleTimeout?.TotalMilliseconds ?? 0), + PrefixPaddingInMilliseconds = (int)(serverVad.PrefixPadding?.TotalMilliseconds ?? 300), + SilenceDurationInMilliseconds = (int)(serverVad.SilenceDuration?.TotalMilliseconds ?? 500), + }; + } + else if (inputOpts.TurnDetection is Sdk.RealtimeSemanticVadTurnDetection semanticVad) + { + vad = new SemanticVoiceActivityDetection + { + CreateResponse = semanticVad.CreateResponseEnabled ?? false, + InterruptResponse = semanticVad.InterruptResponseEnabled ?? false, + Eagerness = semanticVad.EagernessLevel.HasValue + ? new SemanticEagerness(semanticVad.EagernessLevel.Value.ToString()) + : SemanticEagerness.Auto, + }; + } } - if (outputElement.TryGetProperty("voice", out var voiceElement)) + if (audioOptions.OutputAudioOptions is { } outputOpts) { - msg.OutputVoice = voiceElement.GetString(); + outputAudioFormat = MapSdkAudioFormat(outputOpts.AudioFormat); + + if (outputOpts.Speed.HasValue) + { + voiceSpeed = outputOpts.Speed.Value; + } + + if (outputOpts.Voice.HasValue) + { + voice = outputOpts.Voice.Value.ToString(); + } } } - if (responseElement.TryGetProperty("conversation_id", out var conversationIdElement)) + int? maxOutputTokens = null; + if (session.MaxOutputTokenCount is { } maxTokens) { - msg.ConversationId = conversationIdElement.GetString(); + maxOutputTokens = maxTokens.CustomMaxOutputTokenCount ?? int.MaxValue; } - if (responseElement.TryGetProperty("id", out var idElement)) + List? outputModalities = null; + if (session.OutputModalities is { Count: > 0 } modalities) { - msg.ResponseId = idElement.GetString(); + outputModalities = modalities.Select(m => m.ToString()).ToList(); } - if (responseElement.TryGetProperty("max_output_tokens", out var maxOutputTokensElement)) + return new RealtimeSessionOptions + { + SessionKind = RealtimeSessionKind.Realtime, + Model = session.Model, + Instructions = session.Instructions, + MaxOutputTokens = maxOutputTokens, + OutputModalities = outputModalities, + InputAudioFormat = inputAudioFormat, + NoiseReductionOptions = noiseReduction, + TranscriptionOptions = transcription, + VoiceActivityDetection = vad, + OutputAudioFormat = outputAudioFormat, + VoiceSpeed = voiceSpeed, + Voice = voice, + + // Preserve client-side properties that the server cannot round-trip. + Tools = Options?.Tools, + ToolMode = Options?.ToolMode, + }; + } + + private static RealtimeServerResponseCreatedMessage MapResponseCreatedOrDone( + string? eventId, Sdk.RealtimeResponse? response, RealtimeServerMessageType type, Sdk.RealtimeServerUpdate update) + { + var msg = new RealtimeServerResponseCreatedMessage(type) + { + MessageId = eventId, + RawRepresentation = update, + }; + + if (response is null) { - msg.MaxOutputTokens = ParseMaxOutputTokens(maxOutputTokensElement); + return msg; } - if (responseElement.TryGetProperty("metadata", out var metadataElement) && - metadataElement.ValueKind == JsonValueKind.Object) + msg.ResponseId = response.Id; + msg.ConversationId = response.ConversationId; + msg.Status = response.Status?.ToString(); + + if (response.AudioOptions?.OutputAudioOptions is { } audioOut) { - var metadataDict = new AdditionalPropertiesDictionary(); - foreach (var property in metadataElement.EnumerateObject()) + msg.OutputAudioOptions = MapSdkAudioFormat(audioOut.AudioFormat); + if (audioOut.Voice.HasValue) { - metadataDict[property.Name] = property.Value.ValueKind == JsonValueKind.String - ? property.Value.GetString() - : JsonSerializer.Deserialize(property.Value.GetRawText()); + msg.OutputVoice = audioOut.Voice.Value.ToString(); } + } - msg.AdditionalProperties = metadataDict; + if (response.MaxOutputTokenCount is { } maxTokens) + { + msg.MaxOutputTokens = maxTokens.CustomMaxOutputTokenCount ?? int.MaxValue; } - if (responseElement.TryGetProperty("output_modalities", out var outputModalitiesElement)) + if (response.Metadata is { Count: > 0 } metadata) { - msg.OutputModalities = ParseOutputModalities(outputModalitiesElement); + var dict = new AdditionalPropertiesDictionary(); + foreach (var kvp in metadata) + { + dict[kvp.Key] = kvp.Value; + } + + msg.AdditionalProperties = dict; } - if (responseElement.TryGetProperty("status", out var statusElement)) + if (response.OutputModalities is { Count: > 0 } modalities) { - msg.Status = statusElement.GetString(); + msg.OutputModalities = modalities.Select(m => m.ToString()).ToList(); } - if (responseElement.TryGetProperty("status_details", out var statusDetailsElement) && - statusDetailsElement.ValueKind == JsonValueKind.Object && - statusDetailsElement.TryGetProperty("error", out var errorElement) && - errorElement.ValueKind == JsonValueKind.Object && - errorElement.TryGetProperty("type", out var errorTypeElement) && - errorElement.TryGetProperty("code", out var errorCodeElement)) + if (response.StatusDetails?.Error is { } error) { - msg.Error = new ErrorContent(errorTypeElement.GetString()) + msg.Error = new ErrorContent(error.Kind) { - ErrorCode = errorCodeElement.GetString(), + ErrorCode = error.Code, }; } - if (responseElement.TryGetProperty("usage", out var usageElement)) + if (response.Usage is { } usage) { - msg.Usage = ParseUsageDetails(usageElement); + msg.Usage = MapUsageDetails(usage); } - if (responseElement.TryGetProperty("output", out outputElement) && - outputElement.ValueKind == JsonValueKind.Array) + if (response.OutputItems is { Count: > 0 } outputItems) { - var outputItems = new List(); - foreach (var outputItemElement in outputElement.EnumerateArray()) + var items = new List(); + foreach (var item in outputItems) { - if (ParseRealtimeContentItem(outputItemElement) is RealtimeContentItem item) + if (MapRealtimeItem(item) is RealtimeContentItem contentItem) { - outputItems.Add(item); + items.Add(contentItem); } } - msg.Items = outputItems; + msg.Items = items; } return msg; } - private static RealtimeServerResponseOutputItemMessage? CreateMcpCallMessage(JsonElement root, string messageType) + private static RealtimeServerResponseOutputItemMessage MapResponseOutputItem( + string? eventId, string? responseId, int outputIndex, Sdk.RealtimeItem? item, + RealtimeServerMessageType type, Sdk.RealtimeServerUpdate update) { - RealtimeServerMessageType serverMessageType = messageType switch + return new RealtimeServerResponseOutputItemMessage(type) { - "mcp_call.in_progress" => RealtimeServerMessageType.McpCallInProgress, - "mcp_call.completed" => RealtimeServerMessageType.McpCallCompleted, - "mcp_call.failed" => RealtimeServerMessageType.McpCallFailed, - _ => throw new InvalidOperationException($"Unknown message type: {messageType}"), + MessageId = eventId, + ResponseId = responseId, + OutputIndex = outputIndex, + Item = item is not null ? MapRealtimeItem(item) : null, + RawRepresentation = update, }; + } - var msg = new RealtimeServerResponseOutputItemMessage(serverMessageType) + private static RealtimeServerResponseOutputItemMessage MapConversationItem( + string? eventId, Sdk.RealtimeItem? item, RealtimeServerMessageType type, Sdk.RealtimeServerUpdate update) + { + var mapped = item is not null ? MapRealtimeItem(item) : null; + if (mapped is null) { - RawRepresentation = root.Clone(), - }; + return new RealtimeServerResponseOutputItemMessage(RealtimeServerMessageType.RawContentOnly) + { + MessageId = eventId, + RawRepresentation = update, + }; + } - if (root.TryGetProperty("event_id", out var eventIdElement)) + return new RealtimeServerResponseOutputItemMessage(type) { - msg.MessageId = eventIdElement.GetString(); - } + MessageId = eventId, + Item = mapped, + RawRepresentation = update, + }; + } - if (root.TryGetProperty("response_id", out var responseIdElement)) + private static RealtimeServerInputAudioTranscriptionMessage MapInputTranscriptionDelta(Sdk.RealtimeServerUpdateConversationItemInputAudioTranscriptionDelta e) + { + return new RealtimeServerInputAudioTranscriptionMessage(RealtimeServerMessageType.InputAudioTranscriptionDelta) { - msg.ResponseId = responseIdElement.GetString(); - } + MessageId = e.EventId, + ItemId = e.ItemId, + ContentIndex = e.ContentIndex, + Transcription = e.Delta, + RawRepresentation = e, + }; + } - if (root.TryGetProperty("output_index", out var outputIndexElement)) + private static RealtimeServerInputAudioTranscriptionMessage MapInputTranscriptionCompleted(Sdk.RealtimeServerUpdateConversationItemInputAudioTranscriptionCompleted e) + { + return new RealtimeServerInputAudioTranscriptionMessage(RealtimeServerMessageType.InputAudioTranscriptionCompleted) { - msg.OutputIndex = outputIndexElement.GetInt32(); - } + MessageId = e.EventId, + ItemId = e.ItemId, + ContentIndex = e.ContentIndex, + Transcription = e.Transcript, + RawRepresentation = e, + }; + } - if (root.TryGetProperty("item", out var itemElement)) + private static RealtimeServerInputAudioTranscriptionMessage MapInputTranscriptionFailed(Sdk.RealtimeServerUpdateConversationItemInputAudioTranscriptionFailed e) + { + var msg = new RealtimeServerInputAudioTranscriptionMessage(RealtimeServerMessageType.InputAudioTranscriptionFailed) { - msg.Item = ParseRealtimeContentItem(itemElement); - } - else if (root.TryGetProperty("item_id", out var itemIdElement)) + MessageId = e.EventId, + ItemId = e.ItemId, + ContentIndex = e.ContentIndex, + RawRepresentation = e, + }; + + if (e.Error is not null) { - // Some MCP events only include item_id without the full item - msg.Item = new RealtimeContentItem([], itemIdElement.GetString()); + msg.Error = new ErrorContent(e.Error.Message) + { + ErrorCode = e.Error.Code, + Details = e.Error.ParameterName, + }; } return msg; } - private static RealtimeServerResponseOutputItemMessage CreateMcpListToolsMessage(JsonElement root, string messageType) + private static RealtimeServerResponseOutputItemMessage MapMcpCallEvent( + string? eventId, string? itemId, int outputIndex, RealtimeServerMessageType type, Sdk.RealtimeServerUpdate update) { - RealtimeServerMessageType serverMessageType = messageType switch + return new RealtimeServerResponseOutputItemMessage(type) { - "mcp_list_tools.in_progress" => RealtimeServerMessageType.McpListToolsInProgress, - "mcp_list_tools.completed" => RealtimeServerMessageType.McpListToolsCompleted, - "mcp_list_tools.failed" => RealtimeServerMessageType.McpListToolsFailed, - _ => throw new InvalidOperationException($"Unknown message type: {messageType}"), + MessageId = eventId, + Item = itemId is not null ? new RealtimeContentItem([], itemId) : null, + OutputIndex = outputIndex, + RawRepresentation = update, }; + } - var msg = new RealtimeServerResponseOutputItemMessage(serverMessageType) + private static RealtimeServerResponseOutputItemMessage MapMcpListToolsEvent( + string? eventId, string? itemId, RealtimeServerMessageType type, Sdk.RealtimeServerUpdate update) + { + return new RealtimeServerResponseOutputItemMessage(type) { - RawRepresentation = root.Clone(), - MessageId = root.TryGetProperty("event_id", out var eventIdElement) ? eventIdElement.GetString() : null, + MessageId = eventId, + Item = itemId is not null ? new RealtimeContentItem([], itemId) : null, + RawRepresentation = update, }; + } - string? itemId = root.TryGetProperty("item_id", out var itemIdElement) ? itemIdElement.GetString() : null; + private static RealtimeContentItem? MapRealtimeItem(Sdk.RealtimeItem item) => item switch + { + Sdk.RealtimeMessageItem messageItem => MapMessageItem(messageItem), + Sdk.RealtimeFunctionCallItem funcCallItem => MapFunctionCallItem(funcCallItem), + Sdk.RealtimeFunctionCallOutputItem funcOutputItem => new RealtimeContentItem( + [new FunctionResultContent(funcOutputItem.CallId ?? string.Empty, funcOutputItem.FunctionOutput)], + funcOutputItem.Id), + Sdk.RealtimeMcpToolCallItem mcpItem => MapMcpToolCallItem(mcpItem), + Sdk.RealtimeMcpToolCallApprovalRequestItem approvalItem => MapMcpApprovalRequestItem(approvalItem), + Sdk.RealtimeMcpToolDefinitionListItem toolListItem => MapMcpToolDefinitionListItem(toolListItem), + _ => null, + }; - // For completed events, parse the tools list from the item - if (root.TryGetProperty("item", out var itemElement) && itemElement.ValueKind == JsonValueKind.Object) - { - itemId ??= itemElement.TryGetProperty("id", out var idElement) ? idElement.GetString() : null; - string? serverLabel = itemElement.TryGetProperty("server_label", out var slElement) ? slElement.GetString() : null; + private static RealtimeContentItem MapFunctionCallItem(Sdk.RealtimeFunctionCallItem funcCallItem) + { + var arguments = funcCallItem.FunctionArguments is not null && !funcCallItem.FunctionArguments.IsEmpty + ? JsonSerializer.Deserialize>(funcCallItem.FunctionArguments) + : null; + return new RealtimeContentItem( + [new FunctionCallContent(funcCallItem.CallId ?? string.Empty, funcCallItem.FunctionName, arguments)], + funcCallItem.Id); + } - var contents = new List(); - if (itemElement.TryGetProperty("tools", out var toolsArrayElement) && - toolsArrayElement.ValueKind == JsonValueKind.Array) + private static RealtimeContentItem MapMessageItem(Sdk.RealtimeMessageItem messageItem) + { + var contents = new List(); + if (messageItem.Content is not null) + { + foreach (var part in messageItem.Content) { - foreach (var toolElement in toolsArrayElement.EnumerateArray()) + if (part is Sdk.RealtimeInputTextMessageContentPart textPart) + { + contents.Add(new TextContent(textPart.Text)); + } + else if (part is Sdk.RealtimeOutputTextMessageContentPart outputTextPart) + { + contents.Add(new TextContent(outputTextPart.Text)); + } + else if (part is Sdk.RealtimeInputAudioMessageContentPart audioPart) { - string? toolName = toolElement.TryGetProperty("name", out var nameEl) ? nameEl.GetString() : null; - string? toolDesc = toolElement.TryGetProperty("description", out var descEl) ? descEl.GetString() : null; + if (audioPart.AudioBytes is not null) + { + contents.Add(new DataContent($"data:audio/pcm;base64,{Convert.ToBase64String(audioPart.AudioBytes.ToArray())}")); + } + } + else if (part is Sdk.RealtimeOutputAudioMessageContentPart outputAudioPart) + { + if (outputAudioPart.Transcript is not null) + { + contents.Add(new TextContent(outputAudioPart.Transcript)); + } - if (toolName is not null) + if (outputAudioPart.AudioBytes is not null) { - // Represent each discovered tool as an McpServerToolCallContent with no arguments - // so consumers can enumerate tool names and descriptions. - contents.Add(new McpServerToolCallContent(toolName, toolName, serverLabel) - { - RawRepresentation = toolElement.Clone(), - }); + contents.Add(new DataContent($"data:audio/pcm;base64,{Convert.ToBase64String(outputAudioPart.AudioBytes.ToArray())}")); } } + else if (part is Sdk.RealtimeInputImageMessageContentPart imagePart && imagePart.ImageUri is not null) + { + contents.Add(new DataContent(imagePart.ImageUri.ToString())); + } } + } + + ChatRole? role = messageItem.Role == Sdk.RealtimeMessageRole.Assistant ? ChatRole.Assistant + : messageItem.Role == Sdk.RealtimeMessageRole.User ? ChatRole.User + : messageItem.Role == Sdk.RealtimeMessageRole.System ? ChatRole.System + : null; + + return new RealtimeContentItem(contents, messageItem.Id, role); + } + + private static RealtimeContentItem MapMcpToolCallItem(Sdk.RealtimeMcpToolCallItem mcpItem) + { + string callId = mcpItem.Id ?? string.Empty; - msg.Item = new RealtimeContentItem(contents, itemId); + IReadOnlyDictionary? arguments = null; + if (mcpItem.ToolArguments is not null) + { + string argsJson = mcpItem.ToolArguments.ToString(); + if (!string.IsNullOrEmpty(argsJson)) + { + arguments = JsonSerializer.Deserialize>(argsJson); + } } - else if (itemId is not null) + + var contents = new List + { + new McpServerToolCallContent(callId, mcpItem.ToolName ?? string.Empty, mcpItem.ServerLabel) + { + Arguments = arguments, + }, + }; + + // Parse output/error into result content. + if (mcpItem.ToolOutput is not null || mcpItem.Error is not null) { - msg.Item = new RealtimeContentItem([], itemId); + AIContent resultContent = mcpItem.Error is not null + ? new ErrorContent(mcpItem.Error.Message) + : new TextContent(mcpItem.ToolOutput); + + contents.Add(new McpServerToolResultContent(callId) + { + Output = [resultContent], + RawRepresentation = mcpItem, + }); } - return msg; + return new RealtimeContentItem(contents, mcpItem.Id); } - private static RealtimeServerResponseOutputItemMessage? CreateConversationItemMessage(JsonElement root, string messageType) + private static RealtimeContentItem MapMcpApprovalRequestItem(Sdk.RealtimeMcpToolCallApprovalRequestItem approvalItem) { - // conversation.item.added and conversation.item.done carry an "item" property - // which may be an MCP item (mcp_call, mcp_approval_request) or a regular item. - RealtimeServerMessageType serverMessageType = messageType switch + string approvalId = approvalItem.Id ?? string.Empty; + + IReadOnlyDictionary? arguments = null; + if (approvalItem.ToolArguments is not null) + { + string argsJson = approvalItem.ToolArguments.ToString(); + if (!string.IsNullOrEmpty(argsJson)) + { + arguments = JsonSerializer.Deserialize>(argsJson); + } + } + + var toolCall = new McpServerToolCallContent(approvalId, approvalItem.ToolName ?? string.Empty, approvalItem.ServerLabel) { - "conversation.item.added" => RealtimeServerMessageType.ResponseOutputItemAdded, - "conversation.item.done" => RealtimeServerMessageType.ResponseOutputItemDone, - _ => throw new InvalidOperationException($"Unknown message type: {messageType}"), + Arguments = arguments, + RawRepresentation = approvalItem, }; - if (!root.TryGetProperty("item", out var itemElement)) + return new RealtimeContentItem( + [new McpServerToolApprovalRequestContent(approvalId, toolCall) { RawRepresentation = approvalItem }], + approvalItem.Id); + } + + private static RealtimeContentItem MapMcpToolDefinitionListItem(Sdk.RealtimeMcpToolDefinitionListItem toolListItem) + { + var contents = new List(); + foreach (var toolDef in toolListItem.ToolDefinitions) { - return null; + if (toolDef.Name is not null) + { + contents.Add(new McpServerToolCallContent(toolDef.Name, toolDef.Name, toolListItem.ServerLabel) + { + RawRepresentation = toolDef, + }); + } } - var item = ParseRealtimeContentItem(itemElement); - if (item is null) + return new RealtimeContentItem(contents, toolListItem.Id); + } + + private static UsageDetails? MapUsageDetails(Sdk.RealtimeResponseUsage? usage) + { + if (usage is null) { - // If we couldn't parse the item into a typed representation, return as raw - return new RealtimeServerResponseOutputItemMessage(RealtimeServerMessageType.RawContentOnly) - { - RawRepresentation = root.Clone(), - MessageId = root.TryGetProperty("event_id", out var evtElement) ? evtElement.GetString() : null, - }; + return null; } - var msg = new RealtimeServerResponseOutputItemMessage(serverMessageType) + var details = new UsageDetails { - Item = item, - RawRepresentation = root.Clone(), + InputTokenCount = usage.InputTokenCount ?? 0, + OutputTokenCount = usage.OutputTokenCount ?? 0, + TotalTokenCount = usage.TotalTokenCount ?? 0, }; - if (root.TryGetProperty("event_id", out var eventIdElement)) + if (usage.InputTokenDetails is { } inputDetails) { - msg.MessageId = eventIdElement.GetString(); + details.InputAudioTokenCount = inputDetails.AudioTokenCount ?? 0; + details.InputTextTokenCount = inputDetails.TextTokenCount ?? 0; } - return msg; + if (usage.OutputTokenDetails is { } outputDetails) + { + details.OutputAudioTokenCount = outputDetails.AudioTokenCount ?? 0; + details.OutputTextTokenCount = outputDetails.TextTokenCount ?? 0; + } + + return details; } + private static RealtimeAudioFormat? MapSdkAudioFormat(Sdk.RealtimeAudioFormat? format) => format switch + { + Sdk.RealtimePcmAudioFormat pcm => new RealtimeAudioFormat("audio/pcm", pcm.Rate), + Sdk.RealtimePcmuAudioFormat => new RealtimeAudioFormat("audio/pcmu", 8000), + Sdk.RealtimePcmaAudioFormat => new RealtimeAudioFormat("audio/pcma", 8000), + _ => null, + }; + #endregion } diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientTests.cs index 98d24b181e0..3545ad82ab8 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientTests.cs @@ -15,7 +15,7 @@ public class OpenAIRealtimeClientTests [Fact] public void Ctor_InvalidArgs_Throws() { - Assert.Throws("apiKey", () => new OpenAIRealtimeClient(null!, "model")); + Assert.Throws("apiKey", () => new OpenAIRealtimeClient((string)null!, "model")); Assert.Throws("model", () => new OpenAIRealtimeClient("key", null!)); } From 756fd7592da727ee2cfb0a2050b8a3354cee35bf Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 3 Mar 2026 15:00:37 -0800 Subject: [PATCH 40/92] Fix typo: add missing space in comment ('IDsince' -> 'ID since') --- .../ChatCompletion/FunctionInvokingChatClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs index f64d14553b9..3c73b42e966 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs @@ -456,7 +456,7 @@ public override async IAsyncEnumerable GetStreamingResponseA Dictionary? toolMap = null; bool anyToolsRequireApproval = false; - // This is a synthetic IDsince we're generating the tool messages instead of getting them from + // This is a synthetic ID since we're generating the tool messages instead of getting them from // the underlying provider. When emitting the streamed chunks, it's perfectly valid for us to // use the same message ID for all of them within a given iteration, as this is a single logical // message with multiple content items. We could also use different message IDs per tool content, From 5c615409c4498d053bf7660c8c8b2062c38d2d70 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 3 Mar 2026 15:44:37 -0800 Subject: [PATCH 41/92] Make CreateSessionAsync return non-nullable IRealtimeSession - Change IRealtimeClient.CreateSessionAsync return type from Task to Task - Remove exception-swallowing catch blocks in OpenAIRealtimeClient.CreateSessionAsync and OpenAIRealtimeSession.ConnectAsync; let exceptions propagate to callers - Change ConnectAsync from Task to Task - Remove unused System.IO and System.Net.WebSockets usings - Update tests to expect OperationCanceledException instead of null/false returns --- .../Realtime/IRealtimeClient.cs | 2 +- .../OpenAIRealtimeClient.cs | 18 ++++------------- .../OpenAIRealtimeSession.cs | 20 ++++++------------- .../OpenAIRealtimeClientTests.cs | 5 ++--- .../OpenAIRealtimeSessionTests.cs | 5 ++--- 5 files changed, 15 insertions(+), 35 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClient.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClient.cs index b8ccd29d36a..2df6a37dd86 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClient.cs @@ -18,7 +18,7 @@ public interface IRealtimeClient : IDisposable /// The session options. /// A token to cancel the operation. /// The created real-time session. - Task CreateSessionAsync(RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default); + Task CreateSessionAsync(RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default); /// Asks the for an object of the specified type . /// The type of object being requested. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs index efd55da313e..794d9464edd 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs @@ -3,8 +3,6 @@ using System; using System.Diagnostics.CodeAnalysis; -using System.IO; -using System.Net.WebSockets; using System.Threading; using System.Threading.Tasks; using Microsoft.Shared.DiagnosticIds; @@ -54,19 +52,11 @@ public OpenAIRealtimeClient(RealtimeClient realtimeClient, string model) } /// - public async Task CreateSessionAsync(RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default) + public async Task CreateSessionAsync(RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default) { - RealtimeSessionClient sessionClient; - try - { - sessionClient = options?.SessionKind == RealtimeSessionKind.Transcription - ? await _realtimeClient.StartTranscriptionSessionAsync(cancellationToken: cancellationToken).ConfigureAwait(false) - : await _realtimeClient.StartConversationSessionAsync(_model, cancellationToken: cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) when (ex is WebSocketException or OperationCanceledException or IOException) - { - return null; - } + var sessionClient = options?.SessionKind == RealtimeSessionKind.Transcription + ? await _realtimeClient.StartTranscriptionSessionAsync(cancellationToken: cancellationToken).ConfigureAwait(false) + : await _realtimeClient.StartConversationSessionAsync(_model, cancellationToken: cancellationToken).ConfigureAwait(false); var session = new OpenAIRealtimeSession(sessionClient, _model); try diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 380cea2bdeb..d4c42d87bb1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -using System.IO; using System.Linq; using System.Net.WebSockets; using System.Runtime.CompilerServices; @@ -68,24 +67,17 @@ internal OpenAIRealtimeSession(Sdk.RealtimeSessionClient sessionClient, string m /// Connects the WebSocket to the OpenAI Realtime API. /// The to monitor for cancellation requests. - /// if the connection succeeded; otherwise, . - public async Task ConnectAsync(CancellationToken cancellationToken = default) + /// A task representing the asynchronous connect operation. + /// The session was not created with an owned realtime client. + public async Task ConnectAsync(CancellationToken cancellationToken = default) { if (_ownedRealtimeClient is null) { - return false; + Throw.InvalidOperationException("Cannot connect a session that was not created with an owned realtime client."); } - try - { - _sessionClient = await _ownedRealtimeClient.StartConversationSessionAsync( - _model, cancellationToken: cancellationToken).ConfigureAwait(false); - return true; - } - catch (Exception ex) when (ex is WebSocketException or OperationCanceledException or IOException) - { - return false; - } + _sessionClient = await _ownedRealtimeClient.StartConversationSessionAsync( + _model, cancellationToken: cancellationToken).ConfigureAwait(false); } /// diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientTests.cs index 3545ad82ab8..0f39c004f31 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientTests.cs @@ -49,13 +49,12 @@ public void Dispose_CanBeCalledMultipleTimes() } [Fact] - public async Task CreateSessionAsync_Cancelled_ReturnsNull() + public async Task CreateSessionAsync_Cancelled_Throws() { using var client = new OpenAIRealtimeClient("key", "model"); using var cts = new CancellationTokenSource(); cts.Cancel(); - var session = await client.CreateSessionAsync(cancellationToken: cts.Token); - Assert.Null(session); + await Assert.ThrowsAnyAsync(() => client.CreateSessionAsync(cancellationToken: cts.Token)); } } diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs index 6ec0ed0fa8c..b1902b74207 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs @@ -75,13 +75,12 @@ public async Task SendClientMessageAsync_CancelledToken_ReturnsSilently() } [Fact] - public async Task ConnectAsync_CancelledToken_ReturnsFalse() + public async Task ConnectAsync_CancelledToken_Throws() { using var session = new OpenAIRealtimeSession("key", "model"); using var cts = new CancellationTokenSource(); cts.Cancel(); - var result = await session.ConnectAsync(cts.Token); - Assert.False(result); + await Assert.ThrowsAnyAsync(() => session.ConnectAsync(cts.Token)); } } From 82e0b4cfdea5c12168d7958916cee785e014c3e1 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 3 Mar 2026 16:46:17 -0800 Subject: [PATCH 42/92] Remove SemanticEagerness from Abstractions; move eagerness to AdditionalProperties - Delete SemanticEagerness.cs (OpenAI-specific concept) - Remove Eagerness property from SemanticVoiceActivityDetection - Add AdditionalProperties to base VoiceActivityDetection for provider-specific settings - Update OpenAI provider to read/write eagerness via AdditionalProperties["eagerness"] --- .../Realtime/SemanticEagerness.cs | 95 ------------------- .../SemanticVoiceActivityDetection.cs | 4 - .../Realtime/VoiceActivityDetection.cs | 3 + .../OpenAIRealtimeSession.cs | 35 +++++-- 4 files changed, 30 insertions(+), 107 deletions(-) delete mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticEagerness.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticEagerness.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticEagerness.cs deleted file mode 100644 index 3d6cabe4950..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticEagerness.cs +++ /dev/null @@ -1,95 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.ComponentModel; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Text.Json; -using System.Text.Json.Serialization; -using Microsoft.Shared.DiagnosticIds; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Extensions.AI; - -/// -/// Represents the eagerness level for semantic voice activity detection. -/// -[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -[JsonConverter(typeof(Converter))] -[DebuggerDisplay("{Value,nq}")] -public readonly struct SemanticEagerness : IEquatable -{ - /// Gets a value representing low eagerness. - public static SemanticEagerness Low { get; } = new("low"); - - /// Gets a value representing medium eagerness. - public static SemanticEagerness Medium { get; } = new("medium"); - - /// Gets a value representing high eagerness. - public static SemanticEagerness High { get; } = new("high"); - - /// Gets a value representing automatic eagerness detection. - public static SemanticEagerness Auto { get; } = new("auto"); - - /// - /// Gets the value associated with this . - /// - public string Value { get; } - - /// - /// Initializes a new instance of the struct with the provided value. - /// - /// The value to associate with this . - [JsonConstructor] - public SemanticEagerness(string value) - { - Value = Throw.IfNullOrWhitespace(value); - } - - /// - /// Returns a value indicating whether two instances are equivalent, as determined by a - /// case-insensitive comparison of their values. - /// - public static bool operator ==(SemanticEagerness left, SemanticEagerness right) - { - return left.Equals(right); - } - - /// - /// Returns a value indicating whether two instances are not equivalent, as determined by a - /// case-insensitive comparison of their values. - /// - public static bool operator !=(SemanticEagerness left, SemanticEagerness right) - { - return !(left == right); - } - - /// - public override bool Equals([NotNullWhen(true)] object? obj) - => obj is SemanticEagerness other && Equals(other); - - /// - public bool Equals(SemanticEagerness other) - => string.Equals(Value, other.Value, StringComparison.OrdinalIgnoreCase); - - /// - public override int GetHashCode() - => StringComparer.OrdinalIgnoreCase.GetHashCode(Value); - - /// - public override string ToString() => Value; - - /// Provides a for serializing instances. - [EditorBrowsable(EditorBrowsableState.Never)] - public sealed class Converter : JsonConverter - { - /// - public override SemanticEagerness Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => - new(reader.GetString()!); - - /// - public override void Write(Utf8JsonWriter writer, SemanticEagerness value, JsonSerializerOptions options) => - Throw.IfNull(writer).WriteStringValue(value.Value); - } -} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs index c4c94f3b7f5..5f2d3678def 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs @@ -12,8 +12,4 @@ namespace Microsoft.Extensions.AI; [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] public class SemanticVoiceActivityDetection : VoiceActivityDetection { - /// - /// Gets the eagerness level for semantic voice activity detection. - /// - public SemanticEagerness Eagerness { get; init; } = SemanticEagerness.Auto; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs index aa6c58c00f7..c0958f68579 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs @@ -21,4 +21,7 @@ public class VoiceActivityDetection /// Gets a value indicating whether to interrupt the response when voice activity is detected. /// public bool InterruptResponse { get; init; } + + /// Gets or sets any additional properties associated with the voice activity detection options. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index d4c42d87bb1..f9008f2c694 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -437,12 +437,18 @@ private static Sdk.RealtimeConversationSessionOptions BuildConversationSessionOp } else if (options.VoiceActivityDetection is SemanticVoiceActivityDetection semanticVad) { - inputAudioOptions.TurnDetection = new Sdk.RealtimeSemanticVadTurnDetection + var turnDetection = new Sdk.RealtimeSemanticVadTurnDetection { CreateResponseEnabled = semanticVad.CreateResponse, InterruptResponseEnabled = semanticVad.InterruptResponse, - EagernessLevel = new Sdk.RealtimeSemanticVadEagernessLevel(semanticVad.Eagerness.Value), }; + + if (semanticVad.AdditionalProperties?.TryGetValue("eagerness", out var eagerness) is true && eagerness is string eagernessStr) + { + turnDetection.EagernessLevel = new Sdk.RealtimeSemanticVadEagernessLevel(eagernessStr); + } + + inputAudioOptions.TurnDetection = turnDetection; } else if (options.VoiceActivityDetection is { } baseVad) { @@ -558,12 +564,18 @@ private static Sdk.RealtimeTranscriptionSessionOptions BuildTranscriptionSession } else if (options.VoiceActivityDetection is SemanticVoiceActivityDetection semanticVad) { - inputAudioOptions.TurnDetection = new Sdk.RealtimeSemanticVadTurnDetection + var turnDetection = new Sdk.RealtimeSemanticVadTurnDetection { CreateResponseEnabled = semanticVad.CreateResponse, InterruptResponseEnabled = semanticVad.InterruptResponse, - EagernessLevel = new Sdk.RealtimeSemanticVadEagernessLevel(semanticVad.Eagerness.Value), }; + + if (semanticVad.AdditionalProperties?.TryGetValue("eagerness", out var eagerness) is true && eagerness is string eagernessStr) + { + turnDetection.EagernessLevel = new Sdk.RealtimeSemanticVadEagernessLevel(eagernessStr); + } + + inputAudioOptions.TurnDetection = turnDetection; } else if (options.VoiceActivityDetection is { } baseVad) { @@ -949,14 +961,21 @@ private RealtimeSessionOptions MapConversationSessionToOptions(Sdk.RealtimeConve } else if (inputOpts.TurnDetection is Sdk.RealtimeSemanticVadTurnDetection semanticVad) { - vad = new SemanticVoiceActivityDetection + var semanticVadOptions = new SemanticVoiceActivityDetection { CreateResponse = semanticVad.CreateResponseEnabled ?? false, InterruptResponse = semanticVad.InterruptResponseEnabled ?? false, - Eagerness = semanticVad.EagernessLevel.HasValue - ? new SemanticEagerness(semanticVad.EagernessLevel.Value.ToString()) - : SemanticEagerness.Auto, }; + + if (semanticVad.EagernessLevel.HasValue) + { + semanticVadOptions.AdditionalProperties = new AdditionalPropertiesDictionary + { + ["eagerness"] = semanticVad.EagernessLevel.Value.ToString(), + }; + } + + vad = semanticVadOptions; } } From dfb684a7b3f8663b8abe79a4a1a99eb40c352d7f Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 3 Mar 2026 16:47:16 -0800 Subject: [PATCH 43/92] Include CompatibilitySuppressions for SpeechToTextOptions API changes --- .../CompatibilitySuppressions.xml | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/CompatibilitySuppressions.xml b/src/Libraries/Microsoft.Extensions.AI.Abstractions/CompatibilitySuppressions.xml index 2dac6ebccbe..f169dde0a8b 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/CompatibilitySuppressions.xml +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/CompatibilitySuppressions.xml @@ -36,4 +36,144 @@ lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll true + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + true + + + CP0002 + M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + true + \ No newline at end of file From faa6cfbe6ddd5a10889d06a6b25a6b916f7fe559 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 3 Mar 2026 17:30:57 -0800 Subject: [PATCH 44/92] Remove MCP-specific types from abstractions; add provider contract docs --- .../Realtime/RealtimeServerMessageType.cs | 32 ++++++++----------- .../RealtimeServerResponseCreatedMessage.cs | 8 +++++ ...RealtimeServerResponseOutputItemMessage.cs | 8 +++++ .../OpenAIRealtimeSession.cs | 12 +++---- 4 files changed, 36 insertions(+), 24 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs index 7c722cf06a0..455f38631e7 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs @@ -17,8 +17,22 @@ namespace Microsoft.Extensions.AI; /// This is used to identify the message type being received from the model. /// /// +/// /// Well-known message types are provided as static properties. Providers may define additional /// message types by constructing new instances with custom values. +/// +/// +/// Provider implementations that want to support the built-in middleware pipeline +/// ( and +/// ) must emit the following +/// message types at appropriate points during response generation: +/// +/// — when the model begins generating a new response. +/// — when the model has finished generating a response (with usage data if available). +/// — when a new output item (e.g., function call, message) is added during response generation. +/// — when an individual output item has completed. This is required for function invocation middleware to detect and invoke tool calls. +/// +/// /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] [JsonConverter(typeof(Converter))] @@ -73,24 +87,6 @@ namespace Microsoft.Extensions.AI; /// Gets a message type indicating an error occurred while processing the request. public static RealtimeServerMessageType Error { get; } = new("Error"); - /// Gets a message type indicating that an MCP tool call is in progress. - public static RealtimeServerMessageType McpCallInProgress { get; } = new("McpCallInProgress"); - - /// Gets a message type indicating that an MCP tool call has completed. - public static RealtimeServerMessageType McpCallCompleted { get; } = new("McpCallCompleted"); - - /// Gets a message type indicating that an MCP tool call has failed. - public static RealtimeServerMessageType McpCallFailed { get; } = new("McpCallFailed"); - - /// Gets a message type indicating that listing MCP tools is in progress. - public static RealtimeServerMessageType McpListToolsInProgress { get; } = new("McpListToolsInProgress"); - - /// Gets a message type indicating that listing MCP tools has completed. - public static RealtimeServerMessageType McpListToolsCompleted { get; } = new("McpListToolsCompleted"); - - /// Gets a message type indicating that listing MCP tools has failed. - public static RealtimeServerMessageType McpListToolsFailed { get; } = new("McpListToolsFailed"); - /// /// Gets the value associated with this . /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs index 36823ae99e8..d5bc496c4a1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs @@ -11,7 +11,15 @@ namespace Microsoft.Extensions.AI; /// Represents a real-time message for creating a response item. /// /// +/// /// Used with the and messages. +/// +/// +/// Provider implementations should emit this message with +/// when the model begins generating a new response, and with +/// when the response is complete. The built-in middleware depends +/// on these messages for tracing response lifecycle. +/// /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] public class RealtimeServerResponseCreatedMessage : RealtimeServerMessage diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs index bc4e8ff20e6..632d2261005 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs @@ -10,7 +10,15 @@ namespace Microsoft.Extensions.AI; /// Represents a real-time message representing a new output item added or created during response generation. /// /// +/// /// Used with the and messages. +/// +/// +/// Provider implementations should emit this message with +/// when an output item (such as a function call or text message) has completed. The built-in +/// middleware depends on this message to detect +/// and invoke tool calls. +/// /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] public class RealtimeServerResponseOutputItemMessage : RealtimeServerMessage diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index f9008f2c694..99d85be6437 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -865,12 +865,12 @@ private static BinaryData ExtractAudioBinaryData(DataContent content) Sdk.RealtimeServerUpdateConversationItemInputAudioTranscriptionFailed e => MapInputTranscriptionFailed(e), Sdk.RealtimeServerUpdateConversationItemAdded e => MapConversationItem(e.EventId, e.Item, RealtimeServerMessageType.ResponseOutputItemAdded, e), Sdk.RealtimeServerUpdateConversationItemDone e => MapConversationItem(e.EventId, e.Item, RealtimeServerMessageType.ResponseOutputItemDone, e), - Sdk.RealtimeServerUpdateResponseMcpCallInProgress e => MapMcpCallEvent(e.EventId, e.ItemId, e.OutputIndex, RealtimeServerMessageType.McpCallInProgress, e), - Sdk.RealtimeServerUpdateResponseMcpCallCompleted e => MapMcpCallEvent(e.EventId, e.ItemId, e.OutputIndex, RealtimeServerMessageType.McpCallCompleted, e), - Sdk.RealtimeServerUpdateResponseMcpCallFailed e => MapMcpCallEvent(e.EventId, e.ItemId, e.OutputIndex, RealtimeServerMessageType.McpCallFailed, e), - Sdk.RealtimeServerUpdateMcpListToolsInProgress e => MapMcpListToolsEvent(e.EventId, e.ItemId, RealtimeServerMessageType.McpListToolsInProgress, e), - Sdk.RealtimeServerUpdateMcpListToolsCompleted e => MapMcpListToolsEvent(e.EventId, e.ItemId, RealtimeServerMessageType.McpListToolsCompleted, e), - Sdk.RealtimeServerUpdateMcpListToolsFailed e => MapMcpListToolsEvent(e.EventId, e.ItemId, RealtimeServerMessageType.McpListToolsFailed, e), + Sdk.RealtimeServerUpdateResponseMcpCallInProgress e => MapMcpCallEvent(e.EventId, e.ItemId, e.OutputIndex, new RealtimeServerMessageType("McpCallInProgress"), e), + Sdk.RealtimeServerUpdateResponseMcpCallCompleted e => MapMcpCallEvent(e.EventId, e.ItemId, e.OutputIndex, new RealtimeServerMessageType("McpCallCompleted"), e), + Sdk.RealtimeServerUpdateResponseMcpCallFailed e => MapMcpCallEvent(e.EventId, e.ItemId, e.OutputIndex, new RealtimeServerMessageType("McpCallFailed"), e), + Sdk.RealtimeServerUpdateMcpListToolsInProgress e => MapMcpListToolsEvent(e.EventId, e.ItemId, new RealtimeServerMessageType("McpListToolsInProgress"), e), + Sdk.RealtimeServerUpdateMcpListToolsCompleted e => MapMcpListToolsEvent(e.EventId, e.ItemId, new RealtimeServerMessageType("McpListToolsCompleted"), e), + Sdk.RealtimeServerUpdateMcpListToolsFailed e => MapMcpListToolsEvent(e.EventId, e.ItemId, new RealtimeServerMessageType("McpListToolsFailed"), e), _ => new RealtimeServerMessage { Type = RealtimeServerMessageType.RawContentOnly, From 1be4dba55ab9ebebc011f51af9a00de43ecfcb23 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 3 Mar 2026 17:41:22 -0800 Subject: [PATCH 45/92] Remove trailing blank line in RealtimeServerErrorMessage --- .../Realtime/RealtimeServerErrorMessage.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs index ea3131482fa..4c293f5da7b 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs @@ -36,5 +36,4 @@ public RealtimeServerErrorMessage() /// allowing correlation of the error to the originating client request. /// public string? ErrorMessageId { get; set; } - } From 381dc44a6483d648daab87ca4a3001712aed8ca5 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Wed, 4 Mar 2026 10:59:17 -0800 Subject: [PATCH 46/92] Clarify ErrorMessageId XML doc to distinguish from base MessageId --- .../Realtime/RealtimeServerErrorMessage.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs index 4c293f5da7b..e173e1d1089 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs @@ -29,11 +29,11 @@ public RealtimeServerErrorMessage() public ErrorContent? Error { get; set; } /// - /// Gets or sets the message ID of the client message that caused the error. + /// Gets or sets the ID of the client message that caused the error. /// /// - /// This is specific to event-driven protocols where multiple client messages may be in-flight, - /// allowing correlation of the error to the originating client request. + /// Unlike , which identifies this server message itself, + /// this property identifies the originating client message that triggered the error. /// public string? ErrorMessageId { get; set; } } From e186ab8ed4ea31d4a18c8061a944425680f64cd0 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Wed, 4 Mar 2026 11:04:53 -0800 Subject: [PATCH 47/92] Rename ErrorMessageId to OriginatingMessageId for clarity --- .../Realtime/RealtimeServerErrorMessage.cs | 4 ++-- .../Realtime/RealtimeServerMessageTests.cs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs index e173e1d1089..c8a9eac227d 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs @@ -32,8 +32,8 @@ public RealtimeServerErrorMessage() /// Gets or sets the ID of the client message that caused the error. /// /// - /// Unlike , which identifies this server message itself, + /// Unlike , which identifies this server message itself, /// this property identifies the originating client message that triggered the error. /// - public string? ErrorMessageId { get; set; } + public string? OriginatingMessageId { get; set; } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs index fe1c1ef9837..0d70dbb4a77 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs @@ -50,7 +50,7 @@ public void ErrorMessage_DefaultProperties() var message = new RealtimeServerErrorMessage(); Assert.Null(message.Error); - Assert.Null(message.ErrorMessageId); + Assert.Null(message.OriginatingMessageId); } [Fact] @@ -60,12 +60,12 @@ public void ErrorMessage_Properties_Roundtrip() var message = new RealtimeServerErrorMessage { Error = error, - ErrorMessageId = "evt_bad", + OriginatingMessageId = "evt_bad", MessageId = "evt_err_1", }; Assert.Same(error, message.Error); - Assert.Equal("evt_bad", message.ErrorMessageId); + Assert.Equal("evt_bad", message.OriginatingMessageId); Assert.Equal("temperature", message.Error.Details); Assert.Equal("evt_err_1", message.MessageId); Assert.IsAssignableFrom(message); From f752d83f66a93e7eaa8c90d8ae5fc04879de20b5 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Wed, 4 Mar 2026 11:23:31 -0800 Subject: [PATCH 48/92] Change ExcludeFromConversation from bool to bool? for provider-default consistency --- .../Realtime/RealtimeClientResponseCreateMessage.cs | 6 +++--- .../OpenAIRealtimeSession.cs | 9 ++++++--- .../Realtime/RealtimeClientMessageTests.cs | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs index bcd4a016f97..c74e568f2d3 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs @@ -55,10 +55,10 @@ public RealtimeClientResponseCreateMessage() /// /// When , the response is generated out-of-band: the model produces output /// but the resulting items are not added to the conversation history, so they will not appear - /// as context for subsequent responses. Defaults to , meaning response - /// output is added to the default conversation. + /// as context for subsequent responses. + /// If , the provider's default behavior is used. /// - public bool ExcludeFromConversation { get; set; } + public bool? ExcludeFromConversation { get; set; } /// /// Gets or sets the instructions that guide the model on desired responses. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 99d85be6437..76ba4e7d355 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -240,9 +240,12 @@ private async Task SendResponseCreateAsync(RealtimeClientResponseCreateMessage r } // Conversation mode. - responseOptions.DefaultConversationConfiguration = responseCreate.ExcludeFromConversation - ? Sdk.RealtimeResponseDefaultConversationConfiguration.None - : Sdk.RealtimeResponseDefaultConversationConfiguration.Auto; + responseOptions.DefaultConversationConfiguration = responseCreate.ExcludeFromConversation switch + { + true => Sdk.RealtimeResponseDefaultConversationConfiguration.None, + false => Sdk.RealtimeResponseDefaultConversationConfiguration.Auto, + _ => null, + }; // Input items. if (responseCreate.Items is { } items) diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs index f97a521f083..0b2fae34ea7 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs @@ -134,7 +134,7 @@ public void ResponseCreateMessage_DefaultProperties() Assert.Null(message.Items); Assert.Null(message.OutputAudioOptions); Assert.Null(message.OutputVoice); - Assert.False(message.ExcludeFromConversation); + Assert.Null(message.ExcludeFromConversation); Assert.Null(message.Instructions); Assert.Null(message.MaxOutputTokens); Assert.Null(message.AdditionalProperties); From 8a6754ddedccbea0474531c475075fc37c17961d Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Wed, 4 Mar 2026 11:26:35 -0800 Subject: [PATCH 49/92] Remove blank lines between Experimental attribute and class declaration --- .../Realtime/RealtimeClientInputAudioBufferAppendMessage.cs | 1 - .../Realtime/RealtimeClientInputAudioBufferCommitMessage.cs | 1 - 2 files changed, 2 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs index b1b9dd2f038..299e67adf05 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs @@ -11,7 +11,6 @@ namespace Microsoft.Extensions.AI; /// Represents a real-time message for appending audio buffer input. /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] - public class RealtimeClientInputAudioBufferAppendMessage : RealtimeClientMessage { /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs index 15be87316d3..8e7c61a7abc 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs @@ -10,7 +10,6 @@ namespace Microsoft.Extensions.AI; /// Represents a real-time message for committing audio buffer input. /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] - public class RealtimeClientInputAudioBufferCommitMessage : RealtimeClientMessage { /// From 7c5eeb2c59897b2b3136a9d35cfd57c60f799f40 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Wed, 4 Mar 2026 11:33:28 -0800 Subject: [PATCH 50/92] Add null validation to Content property setter --- .../RealtimeClientInputAudioBufferAppendMessage.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs index 299e67adf05..1fc9724c5ee 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs @@ -13,13 +13,15 @@ namespace Microsoft.Extensions.AI; [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] public class RealtimeClientInputAudioBufferAppendMessage : RealtimeClientMessage { + private DataContent _content; + /// /// Initializes a new instance of the class. /// /// The data content containing the audio buffer data to append. public RealtimeClientInputAudioBufferAppendMessage(DataContent audioContent) { - Content = Throw.IfNull(audioContent); + _content = Throw.IfNull(audioContent); } /// @@ -28,5 +30,9 @@ public RealtimeClientInputAudioBufferAppendMessage(DataContent audioContent) /// /// The content should include the audio buffer data that needs to be appended to the input audio buffer. /// - public DataContent Content { get; set; } + public DataContent Content + { + get => _content; + set => _content = Throw.IfNull(value); + } } From d8e8e5ff5115aa611cb3570392ccf7039ebfb6da Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Wed, 4 Mar 2026 11:52:34 -0800 Subject: [PATCH 51/92] Make RealtimeAudioFormat.SampleRate non-nullable and remove misleading doc --- .../Realtime/RealtimeAudioFormat.cs | 6 +----- .../Realtime/RealtimeAudioFormatTests.cs | 11 ----------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs index 3d8962c6780..c8684185268 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeAudioFormat.cs @@ -29,9 +29,5 @@ public RealtimeAudioFormat(string mediaType, int sampleRate) /// /// Gets the sample rate of the audio in Hertz. /// - /// - /// When constructed via , this property is always set. - /// The nullable type allows deserialized instances to omit the sample rate when the server does not provide one. - /// - public int? SampleRate { get; init; } + public int SampleRate { get; init; } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeAudioFormatTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeAudioFormatTests.cs index e6af6ba6b97..bea2c58449e 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeAudioFormatTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeAudioFormatTests.cs @@ -30,15 +30,4 @@ public void Properties_Roundtrip() Assert.Equal("audio/wav", format.MediaType); Assert.Equal(24000, format.SampleRate); } - - [Fact] - public void SampleRate_CanBeSetToNull() - { - var format = new RealtimeAudioFormat("audio/pcm", 16000) - { - SampleRate = null, - }; - - Assert.Null(format.SampleRate); - } } From 621d49dbfcfd3826a3635b35cc639b73e4c95cd9 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Wed, 4 Mar 2026 12:08:10 -0800 Subject: [PATCH 52/92] Rename IRealtimeSession to IRealtimeClientSession and SendClientMessageAsync to SendAsync --- .../Realtime/DelegatingRealtimeSession.cs | 16 +++---- .../Realtime/IRealtimeClient.cs | 2 +- ...meSession.cs => IRealtimeClientSession.cs} | 8 ++-- .../Realtime/RealtimeSessionOptions.cs | 10 ++--- .../OpenAIRealtimeClient.cs | 2 +- .../OpenAIRealtimeSession.cs | 8 ++-- .../AnonymousDelegatingRealtimeSession.cs | 6 +-- .../FunctionInvokingRealtimeSession.cs | 16 +++---- .../Realtime/LoggingRealtimeSession.cs | 20 ++++----- .../Realtime/OpenTelemetryRealtimeSession.cs | 8 ++-- .../Realtime/RealtimeSessionBuilder.cs | 34 +++++++-------- ...SessionBuilderRealtimeSessionExtensions.cs | 4 +- .../Realtime/RealtimeSessionExtensions.cs | 8 ++-- .../TestRealtimeSession.cs | 12 +++--- .../OpenAIRealtimeSessionTests.cs | 16 +++---- .../DelegatingRealtimeSessionTests.cs | 20 ++++----- .../FunctionInvokingRealtimeSessionTests.cs | 26 ++++++------ .../Realtime/LoggingRealtimeSessionTests.cs | 38 ++++++++--------- .../OpenTelemetryRealtimeSessionTests.cs | 42 +++++++++---------- .../Realtime/RealtimeSessionBuilderTests.cs | 16 +++---- .../RealtimeSessionExtensionsTests.cs | 4 +- 21 files changed, 158 insertions(+), 158 deletions(-) rename src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/{IRealtimeSession.cs => IRealtimeClientSession.cs} (89%) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs index 4bb1defb3b9..6267503fe99 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs @@ -12,21 +12,21 @@ namespace Microsoft.Extensions.AI; /// -/// Provides an optional base class for an that passes through calls to another instance. +/// Provides an optional base class for an that passes through calls to another instance. /// /// -/// This is recommended as a base type when building sessions that can be chained around an underlying . +/// This is recommended as a base type when building sessions that can be chained around an underlying . /// The default implementation simply passes each call to the inner session instance. /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class DelegatingRealtimeSession : IRealtimeSession +public class DelegatingRealtimeSession : IRealtimeClientSession { /// /// Initializes a new instance of the class. /// /// The wrapped session instance. /// is . - protected DelegatingRealtimeSession(IRealtimeSession innerSession) + protected DelegatingRealtimeSession(IRealtimeClientSession innerSession) { InnerSession = Throw.IfNull(innerSession); } @@ -61,15 +61,15 @@ protected virtual async ValueTask DisposeAsyncCore() } } - /// Gets the inner . - protected IRealtimeSession InnerSession { get; } + /// Gets the inner . + protected IRealtimeClientSession InnerSession { get; } /// public virtual RealtimeSessionOptions? Options => InnerSession.Options; /// - public virtual Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => - InnerSession.SendClientMessageAsync(message, cancellationToken); + public virtual Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => + InnerSession.SendAsync(message, cancellationToken); /// public virtual Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancellationToken = default) => diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClient.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClient.cs index 2df6a37dd86..5ae142326f1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClient.cs @@ -18,7 +18,7 @@ public interface IRealtimeClient : IDisposable /// The session options. /// A token to cancel the operation. /// The created real-time session. - Task CreateSessionAsync(RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default); + Task CreateSessionAsync(RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default); /// Asks the for an object of the specified type . /// The type of object being requested. diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClientSession.cs similarity index 89% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClientSession.cs index ae27baf91b1..ef1b3e26bc9 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClientSession.cs @@ -13,7 +13,7 @@ namespace Microsoft.Extensions.AI; /// Represents a real-time session. /// This interface provides methods to manage a real-time session and to interact with the real-time model. [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public interface IRealtimeSession : IDisposable, IAsyncDisposable +public interface IRealtimeClientSession : IDisposable, IAsyncDisposable { /// Updates the session with new options. /// The new session options. @@ -35,7 +35,7 @@ public interface IRealtimeSession : IDisposable, IAsyncDisposable /// /// This method allows for sending client messages to the session at any time, which can be used to influence the session's behavior or state. /// - Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default); + Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default); /// Streams the response from the real-time session. /// A token to cancel the operation. @@ -46,13 +46,13 @@ public interface IRealtimeSession : IDisposable, IAsyncDisposable IAsyncEnumerable GetStreamingResponseAsync( CancellationToken cancellationToken = default); - /// Asks the for an object of the specified type . + /// Asks the for an object of the specified type . /// The type of object being requested. /// An optional key that can be used to help identify the target service. /// The found object, otherwise . /// is . /// - /// The purpose of this method is to allow for the retrieval of strongly typed services that might be provided by the , + /// The purpose of this method is to allow for the retrieval of strongly typed services that might be provided by the , /// including itself or any services it might be wrapping. /// object? GetService(Type serviceType, object? serviceKey = null); diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs index 03a02b73187..bff02b1dc0b 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs @@ -94,12 +94,12 @@ public class RealtimeSessionOptions /// Gets a callback responsible for creating the raw representation of the session options from an underlying implementation. /// /// - /// The underlying implementation might have its own representation of options. - /// When is invoked with a , + /// The underlying implementation might have its own representation of options. + /// When is invoked with a , /// that implementation might convert the provided options into its own representation in order to use it while - /// performing the operation. For situations where a consumer knows which concrete + /// performing the operation. For situations where a consumer knows which concrete /// is being used and how it represents options, a new instance of that implementation-specific options type can be - /// returned by this callback for the implementation to use, instead of creating a + /// returned by this callback for the implementation to use, instead of creating a /// new instance. Such implementations might mutate the supplied options instance further based on other settings /// supplied on this instance or from other inputs. /// Therefore, it is strongly recommended to not return shared instances and instead make the callback return @@ -108,5 +108,5 @@ public class RealtimeSessionOptions /// properties on . /// [JsonIgnore] - public Func? RawRepresentationFactory { get; init; } + public Func? RawRepresentationFactory { get; init; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs index 794d9464edd..5a3182547f9 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs @@ -52,7 +52,7 @@ public OpenAIRealtimeClient(RealtimeClient realtimeClient, string model) } /// - public async Task CreateSessionAsync(RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default) + public async Task CreateSessionAsync(RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default) { var sessionClient = options?.SessionKind == RealtimeSessionKind.Transcription ? await _realtimeClient.StartTranscriptionSessionAsync(cancellationToken: cancellationToken).ConfigureAwait(false) diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 76ba4e7d355..9abe60079be 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -23,9 +23,9 @@ namespace Microsoft.Extensions.AI; -/// Represents an for the OpenAI Realtime API over WebSocket. +/// Represents an for the OpenAI Realtime API over WebSocket. [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public sealed class OpenAIRealtimeSession : IRealtimeSession +public sealed class OpenAIRealtimeSession : IRealtimeClientSession { /// The model to use for the session. private readonly string _model; @@ -114,7 +114,7 @@ public async Task UpdateAsync(RealtimeSessionOptions options, CancellationToken } /// - public async Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) + public async Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) { _ = Throw.IfNull(message); @@ -183,7 +183,7 @@ public async IAsyncEnumerable GetStreamingResponseAsync( } /// - object? IRealtimeSession.GetService(Type serviceType, object? serviceKey) + object? IRealtimeClientSession.GetService(Type serviceType, object? serviceKey) { _ = Throw.IfNull(serviceType); diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeSession.cs index 6566f0deb44..6635a72a1d3 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeSession.cs @@ -14,7 +14,7 @@ namespace Microsoft.Extensions.AI; internal sealed class AnonymousDelegatingRealtimeSession : DelegatingRealtimeSession { /// The delegate to use as the implementation of . - private readonly Func> _getStreamingResponseFunc; + private readonly Func> _getStreamingResponseFunc; /// /// Initializes a new instance of the class. @@ -26,8 +26,8 @@ internal sealed class AnonymousDelegatingRealtimeSession : DelegatingRealtimeSes /// is . /// is . public AnonymousDelegatingRealtimeSession( - IRealtimeSession innerSession, - Func> getStreamingResponseFunc) + IRealtimeClientSession innerSession, + Func> getStreamingResponseFunc) : base(innerSession) { _getStreamingResponseFunc = Throw.IfNull(getStreamingResponseFunc); diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs index 899843fbaf1..b9de08e8ea1 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs @@ -28,7 +28,7 @@ namespace Microsoft.Extensions.AI; /// /// /// When this session receives a in a realtime server message from its inner -/// , it responds by invoking the corresponding defined +/// , it responds by invoking the corresponding defined /// in (or in ), producing a /// that it sends back to the inner session. This loop is repeated until there are no more function calls to make, or until /// another stop condition is met, such as hitting . @@ -67,10 +67,10 @@ public class FunctionInvokingRealtimeSession : DelegatingRealtimeSession /// /// Initializes a new instance of the class. /// - /// The underlying , or the next instance in a chain of sessions. + /// The underlying , or the next instance in a chain of sessions. /// An to use for logging information about function invocation. /// An optional to use for resolving services required by the instances being invoked. - public FunctionInvokingRealtimeSession(IRealtimeSession innerSession, ILoggerFactory? loggerFactory = null, IServiceProvider? functionInvocationServices = null) + public FunctionInvokingRealtimeSession(IRealtimeClientSession innerSession, ILoggerFactory? loggerFactory = null, IServiceProvider? functionInvocationServices = null) : base(innerSession) { _logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; @@ -98,11 +98,11 @@ public static FunctionInvocationContext? CurrentContext /// /// Gets or sets a value indicating whether detailed exception information should be included - /// in the response when calling the underlying . + /// in the response when calling the underlying . /// /// /// if the full exception message is added to the response - /// when calling the underlying . + /// when calling the underlying . /// if a generic error message is included in the response. /// The default value is . /// @@ -114,7 +114,7 @@ public static FunctionInvocationContext? CurrentContext /// the property. /// /// - /// Setting the value to can help the underlying bypass problems on + /// Setting the value to can help the underlying bypass problems on /// its own, for example by retrying the function call with different arguments. However it might /// result in disclosing the raw exception information to external users, which can be a security /// concern depending on the application scenario. @@ -186,7 +186,7 @@ public int MaximumIterationsPerRequest /// /// When function invocations fail with an exception, the /// continues to send responses to the inner session, optionally supplying exception information (as - /// controlled by ). This allows the to + /// controlled by ). This allows the to /// recover from errors by trying other function parameters that might succeed. /// /// @@ -305,7 +305,7 @@ public override async IAsyncEnumerable GetStreamingRespon foreach (var resultMessage in results.functionResults) { // inject back the function result messages to the inner session - await InnerSession.SendClientMessageAsync(resultMessage, cancellationToken).ConfigureAwait(false); + await InnerSession.SendAsync(resultMessage, cancellationToken).ConfigureAwait(false); } } } diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs index 34ece9a8df9..12e6183a464 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs @@ -17,7 +17,7 @@ namespace Microsoft.Extensions.AI; /// A delegating realtime session that logs operations to an . /// /// -/// The provided implementation of is thread-safe for concurrent use so long as the +/// The provided implementation of is thread-safe for concurrent use so long as the /// employed is also thread-safe for concurrent use. /// /// @@ -37,9 +37,9 @@ public partial class LoggingRealtimeSession : DelegatingRealtimeSession private JsonSerializerOptions _jsonSerializerOptions; /// Initializes a new instance of the class. - /// The underlying . + /// The underlying . /// An instance that will be used for all logging. - public LoggingRealtimeSession(IRealtimeSession innerSession, ILogger logger) + public LoggingRealtimeSession(IRealtimeClientSession innerSession, ILogger logger) : base(innerSession) { _logger = Throw.IfNull(logger); @@ -90,7 +90,7 @@ public override async Task UpdateAsync(RealtimeSessionOptions options, Cancellat } /// - public override async Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) + public override async Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) { _ = Throw.IfNull(message); @@ -108,21 +108,21 @@ public override async Task SendClientMessageAsync(RealtimeClientMessage message, try { - await base.SendClientMessageAsync(message, cancellationToken).ConfigureAwait(false); + await base.SendAsync(message, cancellationToken).ConfigureAwait(false); if (_logger.IsEnabled(LogLevel.Debug)) { - LogCompleted(nameof(SendClientMessageAsync)); + LogCompleted(nameof(SendAsync)); } } catch (OperationCanceledException) { - LogInvocationCanceled(nameof(SendClientMessageAsync)); + LogInvocationCanceled(nameof(SendAsync)); throw; } catch (Exception ex) { - LogInvocationFailed(nameof(SendClientMessageAsync), ex); + LogInvocationFailed(nameof(SendAsync), ex); throw; } } @@ -254,10 +254,10 @@ private string GetLoggableString(RealtimeServerMessage message) [LoggerMessage(LogLevel.Trace, "{MethodName} invoked: Options: {Options}.")] private partial void LogInvokedSensitive(string methodName, string options); - [LoggerMessage(LogLevel.Debug, "SendClientMessageAsync invoked.")] + [LoggerMessage(LogLevel.Debug, "SendAsync invoked.")] private partial void LogSendMessage(); - [LoggerMessage(LogLevel.Trace, "SendClientMessageAsync invoked: Message: {Message}.")] + [LoggerMessage(LogLevel.Trace, "SendAsync invoked: Message: {Message}.")] private partial void LogSendMessageSensitive(string message); [LoggerMessage(LogLevel.Debug, "{MethodName} completed.")] diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs index 316a107d9f2..1f40b805c45 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs @@ -92,11 +92,11 @@ public sealed partial class OpenTelemetryRealtimeSession : DelegatingRealtimeSes private JsonSerializerOptions _jsonSerializerOptions; /// Initializes a new instance of the class. - /// The underlying . + /// The underlying . /// The to use for emitting any logging data from the session. /// An optional source name that will be used on the telemetry data. #pragma warning disable IDE0060 // Remove unused parameter; it exists for backwards compatibility and future use - public OpenTelemetryRealtimeSession(IRealtimeSession innerSession, ILogger? logger = null, string? sourceName = null) + public OpenTelemetryRealtimeSession(IRealtimeClientSession innerSession, ILogger? logger = null, string? sourceName = null) #pragma warning restore IDE0060 : base(innerSession) { @@ -200,7 +200,7 @@ public override async Task UpdateAsync(RealtimeSessionOptions options, Cancellat } /// - public override async Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) + public override async Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) { if (EnableSensitiveData && _activitySource.HasListeners()) { @@ -228,7 +228,7 @@ public override async Task SendClientMessageAsync(RealtimeClientMessage message, } } - await base.SendClientMessageAsync(message, cancellationToken).ConfigureAwait(false); + await base.SendAsync(message, cancellationToken).ConfigureAwait(false); } /// diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilder.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilder.cs index de02b3dbe1d..3240c11c0c4 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilder.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilder.cs @@ -9,38 +9,38 @@ namespace Microsoft.Extensions.AI; -/// A builder for creating pipelines of . +/// A builder for creating pipelines of . [Experimental("MEAI001")] public sealed class RealtimeSessionBuilder { - private readonly Func _innerSessionFactory; + private readonly Func _innerSessionFactory; /// The registered session factory instances. - private List>? _sessionFactories; + private List>? _sessionFactories; /// Initializes a new instance of the class. - /// The inner that represents the underlying backend. + /// The inner that represents the underlying backend. /// is . - public RealtimeSessionBuilder(IRealtimeSession innerSession) + public RealtimeSessionBuilder(IRealtimeClientSession innerSession) { _ = Throw.IfNull(innerSession); _innerSessionFactory = _ => innerSession; } /// Initializes a new instance of the class. - /// A callback that produces the inner that represents the underlying backend. - public RealtimeSessionBuilder(Func innerSessionFactory) + /// A callback that produces the inner that represents the underlying backend. + public RealtimeSessionBuilder(Func innerSessionFactory) { _innerSessionFactory = Throw.IfNull(innerSessionFactory); } - /// Builds an that represents the entire pipeline. Calls to this instance will pass through each of the pipeline stages in turn. + /// Builds an that represents the entire pipeline. Calls to this instance will pass through each of the pipeline stages in turn. /// - /// The that should provide services to the instances. + /// The that should provide services to the instances. /// If , an empty will be used. /// - /// An instance of that represents the entire pipeline. - public IRealtimeSession Build(IServiceProvider? services = null) + /// An instance of that represents the entire pipeline. + public IRealtimeClientSession Build(IServiceProvider? services = null) { services ??= EmptyServiceProvider.Instance; var session = _innerSessionFactory(services); @@ -55,7 +55,7 @@ public IRealtimeSession Build(IServiceProvider? services = null) { Throw.InvalidOperationException( $"The {nameof(RealtimeSessionBuilder)} entry at index {i} returned null. " + - $"Ensure that the callbacks passed to {nameof(Use)} return non-null {nameof(IRealtimeSession)} instances."); + $"Ensure that the callbacks passed to {nameof(Use)} return non-null {nameof(IRealtimeClientSession)} instances."); } } } @@ -67,7 +67,7 @@ public IRealtimeSession Build(IServiceProvider? services = null) /// The session factory function. /// The updated instance. /// is . - public RealtimeSessionBuilder Use(Func sessionFactory) + public RealtimeSessionBuilder Use(Func sessionFactory) { _ = Throw.IfNull(sessionFactory); @@ -78,7 +78,7 @@ public RealtimeSessionBuilder Use(Func sessi /// The session factory function. /// The updated instance. /// is . - public RealtimeSessionBuilder Use(Func sessionFactory) + public RealtimeSessionBuilder Use(Func sessionFactory) { _ = Throw.IfNull(sessionFactory); @@ -88,10 +88,10 @@ public RealtimeSessionBuilder Use(Func /// Adds to the realtime session pipeline an anonymous delegating realtime session based on a delegate that provides - /// an implementation for . + /// an implementation for . /// /// - /// A delegate that provides the implementation for . + /// A delegate that provides the implementation for . /// This delegate is invoked with a delegate that represents invoking /// the inner session, and a cancellation token. The delegate should be passed whatever /// cancellation token should be passed along to the next stage in the pipeline. @@ -103,7 +103,7 @@ public RealtimeSessionBuilder Use(Func /// is . public RealtimeSessionBuilder Use( - Func> getStreamingResponseFunc) + Func> getStreamingResponseFunc) { _ = Throw.IfNull(getStreamingResponseFunc); diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilderRealtimeSessionExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilderRealtimeSessionExtensions.cs index 1bf0bd8d489..390043243c9 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilderRealtimeSessionExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilderRealtimeSessionExtensions.cs @@ -7,7 +7,7 @@ namespace Microsoft.Extensions.AI; -/// Provides extension methods for working with in the context of . +/// Provides extension methods for working with in the context of . [Experimental("MEAI001")] public static class RealtimeSessionBuilderRealtimeSessionExtensions { @@ -19,7 +19,7 @@ public static class RealtimeSessionBuilderRealtimeSessionExtensions /// specifying as the inner session. /// /// is . - public static RealtimeSessionBuilder AsBuilder(this IRealtimeSession innerSession) + public static RealtimeSessionBuilder AsBuilder(this IRealtimeClientSession innerSession) { _ = Throw.IfNull(innerSession); diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionExtensions.cs index fe3f95fef19..0063b0d5c0d 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionExtensions.cs @@ -7,21 +7,21 @@ namespace Microsoft.Extensions.AI; -/// Provides a collection of static methods for extending instances. +/// Provides a collection of static methods for extending instances. [Experimental("MEAI001")] public static class RealtimeSessionExtensions { - /// Asks the for an object of type . + /// Asks the for an object of type . /// The type of the object to be retrieved. /// The session. /// An optional key that can be used to help identify the target service. /// The found object, otherwise . /// is . /// - /// The purpose of this method is to allow for the retrieval of strongly typed services that may be provided by the , + /// The purpose of this method is to allow for the retrieval of strongly typed services that may be provided by the , /// including itself or any services it might be wrapping. /// - public static TService? GetService(this IRealtimeSession session, object? serviceKey = null) + public static TService? GetService(this IRealtimeClientSession session, object? serviceKey = null) { _ = Throw.IfNull(session); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs index 4ec5d6e03a8..69cb64a9bd8 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs @@ -8,14 +8,14 @@ namespace Microsoft.Extensions.AI; -/// A test implementation that uses callbacks for verification. -public sealed class TestRealtimeSession : IRealtimeSession +/// A test implementation that uses callbacks for verification. +public sealed class TestRealtimeSession : IRealtimeClientSession { /// Gets or sets the callback to invoke when is called. public Func? UpdateAsyncCallback { get; set; } - /// Gets or sets the callback to invoke when is called. - public Func? SendClientMessageAsyncCallback { get; set; } + /// Gets or sets the callback to invoke when is called. + public Func? SendAsyncCallback { get; set; } /// Gets or sets the callback to invoke when is called. public Func>? GetStreamingResponseAsyncCallback { get; set; } @@ -33,9 +33,9 @@ public Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancel } /// - public Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) + public Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) { - return SendClientMessageAsyncCallback?.Invoke(message, cancellationToken) ?? Task.CompletedTask; + return SendAsyncCallback?.Invoke(message, cancellationToken) ?? Task.CompletedTask; } /// diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs index b1902b74207..2418f682319 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs @@ -15,10 +15,10 @@ public class OpenAIRealtimeSessionTests [Fact] public void GetService_ReturnsExpectedServices() { - using IRealtimeSession session = new OpenAIRealtimeSession("key", "model"); + using IRealtimeClientSession session = new OpenAIRealtimeSession("key", "model"); Assert.Same(session, session.GetService(typeof(OpenAIRealtimeSession))); - Assert.Same(session, session.GetService(typeof(IRealtimeSession))); + Assert.Same(session, session.GetService(typeof(IRealtimeClientSession))); Assert.Null(session.GetService(typeof(string))); Assert.Null(session.GetService(typeof(OpenAIRealtimeSession), "someKey")); } @@ -26,14 +26,14 @@ public void GetService_ReturnsExpectedServices() [Fact] public void GetService_NullServiceType_Throws() { - using IRealtimeSession session = new OpenAIRealtimeSession("key", "model"); + using IRealtimeClientSession session = new OpenAIRealtimeSession("key", "model"); Assert.Throws("serviceType", () => session.GetService(null!)); } [Fact] public void Dispose_CanBeCalledMultipleTimes() { - IRealtimeSession session = new OpenAIRealtimeSession("key", "model"); + IRealtimeClientSession session = new OpenAIRealtimeSession("key", "model"); session.Dispose(); // Second dispose should not throw. @@ -56,21 +56,21 @@ public async Task UpdateAsync_NullOptions_Throws() } [Fact] - public async Task SendClientMessageAsync_NullMessage_Throws() + public async Task SendAsync_NullMessage_Throws() { using var session = new OpenAIRealtimeSession("key", "model"); - await Assert.ThrowsAsync("message", () => session.SendClientMessageAsync(null!)); + await Assert.ThrowsAsync("message", () => session.SendAsync(null!)); } [Fact] - public async Task SendClientMessageAsync_CancelledToken_ReturnsSilently() + public async Task SendAsync_CancelledToken_ReturnsSilently() { using var session = new OpenAIRealtimeSession("key", "model"); using var cts = new CancellationTokenSource(); cts.Cancel(); // Should not throw when cancellation is requested. - await session.SendClientMessageAsync(new RealtimeClientMessage(), cts.Token); + await session.SendAsync(new RealtimeClientMessage(), cts.Token); Assert.Null(session.Options); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs index 9241a02fbbd..04e80e22023 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs @@ -51,13 +51,13 @@ public async Task UpdateAsync_DelegatesToInner() } [Fact] - public async Task SendClientMessageAsync_DelegatesToInner() + public async Task SendAsync_DelegatesToInner() { var called = false; var sentMessage = new RealtimeClientMessage { MessageId = "evt_001" }; using var inner = new TestRealtimeSession { - SendClientMessageAsyncCallback = (msg, _) => + SendAsyncCallback = (msg, _) => { Assert.Same(sentMessage, msg); called = true; @@ -66,7 +66,7 @@ public async Task SendClientMessageAsync_DelegatesToInner() }; using var delegating = new NoOpDelegatingRealtimeSession(inner); - await delegating.SendClientMessageAsync(sentMessage); + await delegating.SendAsync(sentMessage); Assert.True(called); } @@ -98,7 +98,7 @@ public void GetService_ReturnsSelfForMatchingType() Assert.Same(delegating, delegating.GetService(typeof(NoOpDelegatingRealtimeSession))); Assert.Same(delegating, delegating.GetService(typeof(DelegatingRealtimeSession))); - Assert.Same(delegating, delegating.GetService(typeof(IRealtimeSession))); + Assert.Same(delegating, delegating.GetService(typeof(IRealtimeClientSession))); } [Fact] @@ -164,7 +164,7 @@ public async Task UpdateAsync_FlowsCancellationToken() } [Fact] - public async Task SendClientMessageAsync_FlowsCancellationToken() + public async Task SendAsync_FlowsCancellationToken() { CancellationToken capturedToken = default; using var cts = new CancellationTokenSource(); @@ -172,7 +172,7 @@ public async Task SendClientMessageAsync_FlowsCancellationToken() using var inner = new TestRealtimeSession { - SendClientMessageAsyncCallback = (msg, ct) => + SendAsyncCallback = (msg, ct) => { capturedToken = ct; return Task.CompletedTask; @@ -180,7 +180,7 @@ public async Task SendClientMessageAsync_FlowsCancellationToken() }; using var delegating = new NoOpDelegatingRealtimeSession(inner); - await delegating.SendClientMessageAsync(sentMessage, cts.Token); + await delegating.SendAsync(sentMessage, cts.Token); Assert.Equal(cts.Token, capturedToken); } @@ -196,14 +196,14 @@ private static async IAsyncEnumerable YieldSingle( /// A concrete DelegatingRealtimeSession for testing (since the base class is abstract-ish with protected ctor). private sealed class NoOpDelegatingRealtimeSession : DelegatingRealtimeSession { - public NoOpDelegatingRealtimeSession(IRealtimeSession innerSession) + public NoOpDelegatingRealtimeSession(IRealtimeClientSession innerSession) : base(innerSession) { } } /// A test session that tracks Dispose calls. - private sealed class DisposableTestRealtimeSession : IRealtimeSession + private sealed class DisposableTestRealtimeSession : IRealtimeClientSession { private readonly Action _onDispose; @@ -216,7 +216,7 @@ public DisposableTestRealtimeSession(Action onDispose) public Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancellationToken = default) => Task.CompletedTask; - public Task SendClientMessageAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => Task.CompletedTask; + public Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => Task.CompletedTask; public IAsyncEnumerable GetStreamingResponseAsync( CancellationToken cancellationToken = default) => diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs index 173cda9ee60..74cc46c4866 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs @@ -103,7 +103,7 @@ public async Task GetStreamingResponseAsync_FunctionCall_InvokesAndInjectsResult [ CreateFunctionCallOutputItemMessage("call_001", "get_weather", new Dictionary { ["city"] = "Seattle" }), ], ct), - SendClientMessageAsyncCallback = (msg, _) => + SendAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -151,7 +151,7 @@ public async Task GetStreamingResponseAsync_FunctionCall_FromAdditionalTools() [ CreateFunctionCallOutputItemMessage("call_002", "get_weather", new Dictionary { ["city"] = "London" }), ], ct), - SendClientMessageAsyncCallback = (msg, _) => + SendAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -194,7 +194,7 @@ public async Task GetStreamingResponseAsync_MaxIterations_StopsInvoking() { Options = new RealtimeSessionOptions { Tools = [countFunc] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages(messages, ct), - SendClientMessageAsyncCallback = (_, _) => Task.CompletedTask, + SendAsyncCallback = (_, _) => Task.CompletedTask, }; using var session = new FunctionInvokingRealtimeSession(inner) @@ -231,7 +231,7 @@ public async Task GetStreamingResponseAsync_FunctionInvoker_CustomDelegate() [ CreateFunctionCallOutputItemMessage("call_custom", "my_func", null), ], ct), - SendClientMessageAsyncCallback = (_, _) => Task.CompletedTask, + SendAsyncCallback = (_, _) => Task.CompletedTask, }; using var session = new FunctionInvokingRealtimeSession(inner) @@ -261,7 +261,7 @@ public async Task GetStreamingResponseAsync_UnknownFunction_SendsErrorByDefault( [ CreateFunctionCallOutputItemMessage("call_unknown", "nonexistent_func", null), ], ct), - SendClientMessageAsyncCallback = (msg, _) => + SendAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -298,7 +298,7 @@ public async Task GetStreamingResponseAsync_FunctionError_IncludesDetailedErrors [ CreateFunctionCallOutputItemMessage("call_fail", "fail_func", null), ], ct), - SendClientMessageAsyncCallback = (msg, _) => + SendAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -337,7 +337,7 @@ public async Task GetStreamingResponseAsync_FunctionError_HidesDetailsWhenNotEna [ CreateFunctionCallOutputItemMessage("call_fail2", "fail_func", null), ], ct), - SendClientMessageAsyncCallback = (msg, _) => + SendAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -367,7 +367,7 @@ public void GetService_ReturnsSelf() using var session = new FunctionInvokingRealtimeSession(inner); Assert.Same(session, session.GetService(typeof(FunctionInvokingRealtimeSession))); - Assert.Same(session, session.GetService(typeof(IRealtimeSession))); + Assert.Same(session, session.GetService(typeof(IRealtimeClientSession))); Assert.Same(inner, session.GetService(typeof(TestRealtimeSession))); } @@ -382,7 +382,7 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_StopsLoop() CreateFunctionCallOutputItemMessage("call_unknown", "nonexistent_func", null), new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseDone, MessageId = "should_not_reach" }, ], ct), - SendClientMessageAsyncCallback = (msg, _) => + SendAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -417,7 +417,7 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_False_SendsE [ CreateFunctionCallOutputItemMessage("call_unknown", "nonexistent_func", null), ], ct), - SendClientMessageAsyncCallback = (msg, _) => + SendAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; @@ -499,7 +499,7 @@ public async Task GetStreamingResponseAsync_ConcurrentInvocation_InvokesInParall { Options = new RealtimeSessionOptions { Tools = [slowFunc] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages([combinedMessage], ct), - SendClientMessageAsyncCallback = (_, _) => Task.CompletedTask, + SendAsyncCallback = (_, _) => Task.CompletedTask, }; using var session = new FunctionInvokingRealtimeSession(inner) @@ -542,7 +542,7 @@ public async Task GetStreamingResponseAsync_ConsecutiveErrors_ExceedsLimit_Throw { Options = new RealtimeSessionOptions { Tools = [failFunc] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages(messages, ct), - SendClientMessageAsyncCallback = (_, _) => Task.CompletedTask, + SendAsyncCallback = (_, _) => Task.CompletedTask, }; using var session = new FunctionInvokingRealtimeSession(inner) @@ -608,7 +608,7 @@ public async Task GetStreamingResponseAsync_NonInvocableTool_TerminatesLoop() CreateFunctionCallOutputItemMessage("call_decl", "my_declaration", null), new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseDone, MessageId = "should_not_reach" }, ], ct), - SendClientMessageAsyncCallback = (msg, _) => + SendAsyncCallback = (msg, _) => { injectedMessages.Add(msg); return Task.CompletedTask; diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs index 9eac51a59ea..0a866631ee4 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs @@ -30,7 +30,7 @@ public void UseLogging_AvoidsInjectingNopSession() using var innerSession = new TestRealtimeSession(); Assert.Null(innerSession.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build().GetService(typeof(LoggingRealtimeSession))); - Assert.Same(innerSession, innerSession.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build().GetService(typeof(IRealtimeSession))); + Assert.Same(innerSession, innerSession.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build().GetService(typeof(IRealtimeClientSession))); using var factory = LoggerFactory.Create(b => b.AddFakeLogging()); Assert.NotNull(innerSession.AsBuilder().UseLogging(factory).Build().GetService(typeof(LoggingRealtimeSession))); @@ -90,7 +90,7 @@ public async Task UpdateAsync_LogsInvocationAndCompletion(LogLevel level) [InlineData(LogLevel.Trace)] [InlineData(LogLevel.Debug)] [InlineData(LogLevel.Information)] - public async Task SendClientMessageAsync_LogsInvocationAndCompletion(LogLevel level) + public async Task SendAsync_LogsInvocationAndCompletion(LogLevel level) { var collector = new FakeLogCollector(); @@ -100,7 +100,7 @@ public async Task SendClientMessageAsync_LogsInvocationAndCompletion(LogLevel le using var innerSession = new TestRealtimeSession { - SendClientMessageAsyncCallback = (message, cancellationToken) => Task.CompletedTask, + SendAsyncCallback = (message, cancellationToken) => Task.CompletedTask, }; using var session = innerSession @@ -108,20 +108,20 @@ public async Task SendClientMessageAsync_LogsInvocationAndCompletion(LogLevel le .UseLogging() .Build(services); - await session.SendClientMessageAsync(new RealtimeClientMessage { MessageId = "test-event-123" }); + await session.SendAsync(new RealtimeClientMessage { MessageId = "test-event-123" }); var logs = collector.GetSnapshot(); if (level is LogLevel.Trace) { Assert.Collection(logs, - entry => Assert.Contains("SendClientMessageAsync invoked:", entry.Message), - entry => Assert.Contains("SendClientMessageAsync completed.", entry.Message)); + entry => Assert.Contains("SendAsync invoked:", entry.Message), + entry => Assert.Contains("SendAsync completed.", entry.Message)); } else if (level is LogLevel.Debug) { Assert.Collection(logs, - entry => Assert.Contains("SendClientMessageAsync invoked.", entry.Message), - entry => Assert.Contains("SendClientMessageAsync completed.", entry.Message)); + entry => Assert.Contains("SendAsync invoked.", entry.Message), + entry => Assert.Contains("SendAsync completed.", entry.Message)); } else { @@ -334,11 +334,11 @@ public void GetService_ReturnsLoggingSessionWhenRequested() .Build(); Assert.NotNull(session.GetService(typeof(LoggingRealtimeSession))); - Assert.Same(session, session.GetService(typeof(IRealtimeSession))); + Assert.Same(session, session.GetService(typeof(IRealtimeClientSession))); } [Fact] - public async Task SendClientMessageAsync_LogsCancellation() + public async Task SendAsync_LogsCancellation() { var collector = new FakeLogCollector(); using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); @@ -347,7 +347,7 @@ public async Task SendClientMessageAsync_LogsCancellation() using var innerSession = new TestRealtimeSession { - SendClientMessageAsyncCallback = (message, cancellationToken) => + SendAsyncCallback = (message, cancellationToken) => { throw new OperationCanceledException(cancellationToken); }, @@ -360,23 +360,23 @@ public async Task SendClientMessageAsync_LogsCancellation() cts.Cancel(); await Assert.ThrowsAsync(() => - session.SendClientMessageAsync(new RealtimeClientMessage { MessageId = "evt_cancel" }, cts.Token)); + session.SendAsync(new RealtimeClientMessage { MessageId = "evt_cancel" }, cts.Token)); var logs = collector.GetSnapshot(); Assert.Collection(logs, - entry => Assert.Contains("SendClientMessageAsync invoked.", entry.Message), - entry => Assert.Contains("SendClientMessageAsync canceled.", entry.Message)); + entry => Assert.Contains("SendAsync invoked.", entry.Message), + entry => Assert.Contains("SendAsync canceled.", entry.Message)); } [Fact] - public async Task SendClientMessageAsync_LogsErrors() + public async Task SendAsync_LogsErrors() { var collector = new FakeLogCollector(); using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); using var innerSession = new TestRealtimeSession { - SendClientMessageAsyncCallback = (message, cancellationToken) => + SendAsyncCallback = (message, cancellationToken) => { throw new InvalidOperationException("Inject error"); }, @@ -388,12 +388,12 @@ public async Task SendClientMessageAsync_LogsErrors() .Build(); await Assert.ThrowsAsync(() => - session.SendClientMessageAsync(new RealtimeClientMessage())); + session.SendAsync(new RealtimeClientMessage())); var logs = collector.GetSnapshot(); Assert.Collection(logs, - entry => Assert.Contains("SendClientMessageAsync invoked.", entry.Message), - entry => Assert.True(entry.Message.Contains("SendClientMessageAsync failed.") && entry.Level == LogLevel.Error)); + entry => Assert.Contains("SendAsync invoked.", entry.Message), + entry => Assert.True(entry.Message.Contains("SendAsync failed.") && entry.Level == LogLevel.Error)); } [Fact] diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs index 5494b202a3a..7b6023e2e03 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs @@ -87,7 +87,7 @@ static async IAsyncEnumerable CallbackAsync([EnumeratorCa await foreach (var msg in GetClientMessagesAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -339,7 +339,7 @@ static async IAsyncEnumerable ErrorResponseCallbackAsync( await foreach (var msg in GetClientMessagesAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -388,7 +388,7 @@ static async IAsyncEnumerable EmptyCallbackAsync([Enumera await foreach (var msg in GetClientMessagesAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -480,7 +480,7 @@ public void GetService_ReturnsSelf() var self = session.GetService(typeof(OpenTelemetryRealtimeSession)); Assert.Same(session, self); - var realtime = session.GetService(typeof(IRealtimeSession)); + var realtime = session.GetService(typeof(IRealtimeClientSession)); Assert.Same(session, realtime); } @@ -523,7 +523,7 @@ static async IAsyncEnumerable TranscriptionCallbackAsync( await foreach (var msg in GetClientMessagesAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -574,7 +574,7 @@ public async Task ToolChoiceMode_Logged(string modeKey, string expectedValue) await foreach (var msg in GetClientMessagesAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -613,7 +613,7 @@ public async Task AIFunction_ForcedTool_Logged() await foreach (var msg in GetClientMessagesAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -653,7 +653,7 @@ public async Task RequireAny_ToolMode_Logged() await foreach (var msg in GetClientMessagesAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -692,7 +692,7 @@ public async Task NoToolChoice_NotLogged() await foreach (var msg in GetClientMessagesAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -729,7 +729,7 @@ public async Task ToolCallContentInClientMessages_LoggedAsInputMessages() await foreach (var msg in GetClientMessagesWithToolResultAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -772,7 +772,7 @@ public async Task ToolCallContentInServerMessages_LoggedAsOutputMessages() await foreach (var msg in GetClientMessagesAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -815,7 +815,7 @@ public async Task ToolContentNotLoggedWithoutSensitiveData() await foreach (var msg in GetClientMessagesWithToolResultAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -900,7 +900,7 @@ public async Task AudioBufferAppendMessage_LoggedAsInputMessage() await foreach (var msg in GetClientMessagesAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -943,7 +943,7 @@ public async Task AudioBufferCommitMessage_LoggedAsInputMessage() await foreach (var msg in GetClientMessagesAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -986,7 +986,7 @@ public async Task ResponseCreateMessageWithInstructions_LoggedAsInputMessage() await foreach (var msg in GetClientMessagesWithInstructionsAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -1028,7 +1028,7 @@ public async Task ResponseCreateMessageWithItems_LoggedAsInputMessage() await foreach (var msg in GetClientMessagesWithItemsAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -1070,7 +1070,7 @@ public async Task OutputTextAudioMessage_LoggedAsOutputMessage() await foreach (var msg in GetClientMessagesAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -1111,7 +1111,7 @@ public async Task InputAudioTranscriptionMessage_LoggedAsOutputMessage() await foreach (var msg in GetClientMessagesAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -1152,7 +1152,7 @@ public async Task ServerErrorMessage_LoggedAsOutputMessage() await foreach (var msg in GetClientMessagesAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -1193,7 +1193,7 @@ public async Task ConversationItemCreateWithTextContent_LoggedAsInputMessage() await foreach (var msg in GetClientMessagesWithTextContentAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) @@ -1234,7 +1234,7 @@ public async Task DataContentInClientMessage_LoggedWithModality() await foreach (var msg in GetClientMessagesWithImageContentAsync()) { - await session.SendClientMessageAsync(msg); + await session.SendAsync(msg); } await foreach (var response in session.GetStreamingResponseAsync()) diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs index b39e698c57f..4caa268c438 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs @@ -17,13 +17,13 @@ public class RealtimeSessionBuilderTests [Fact] public void Ctor_NullSession_Throws() { - Assert.Throws("innerSession", () => new RealtimeSessionBuilder((IRealtimeSession)null!)); + Assert.Throws("innerSession", () => new RealtimeSessionBuilder((IRealtimeClientSession)null!)); } [Fact] public void Ctor_NullFactory_Throws() { - Assert.Throws("innerSessionFactory", () => new RealtimeSessionBuilder((Func)null!)); + Assert.Throws("innerSessionFactory", () => new RealtimeSessionBuilder((Func)null!)); } [Fact] @@ -52,8 +52,8 @@ public void Use_NullSessionFactory_Throws() using var inner = new TestRealtimeSession(); var builder = new RealtimeSessionBuilder(inner); - Assert.Throws("sessionFactory", () => builder.Use((Func)null!)); - Assert.Throws("sessionFactory", () => builder.Use((Func)null!)); + Assert.Throws("sessionFactory", () => builder.Use((Func)null!)); + Assert.Throws("sessionFactory", () => builder.Use((Func)null!)); } [Fact] @@ -64,7 +64,7 @@ public void Use_StreamingDelegate_NullFunc_Throws() Assert.Throws( "getStreamingResponseFunc", - () => builder.Use((Func>)null!)); + () => builder.Use((Func>)null!)); } [Fact] @@ -164,7 +164,7 @@ public async Task Use_WithStreamingDelegate_InterceptsStreaming() [Fact] public void AsBuilder_NullSession_Throws() { - Assert.Throws("innerSession", () => ((IRealtimeSession)null!).AsBuilder()); + Assert.Throws("innerSession", () => ((IRealtimeClientSession)null!).AsBuilder()); } [Fact] @@ -191,14 +191,14 @@ private sealed class OrderTrackingSession : DelegatingRealtimeSession public string Name { get; } private readonly List _callOrder; - public OrderTrackingSession(IRealtimeSession inner, string name, List callOrder) + public OrderTrackingSession(IRealtimeClientSession inner, string name, List callOrder) : base(inner) { Name = name; _callOrder = callOrder; } - public IRealtimeSession GetInner() => InnerSession; + public IRealtimeClientSession GetInner() => InnerSession; public override async IAsyncEnumerable GetStreamingResponseAsync( [EnumeratorCancellation] CancellationToken cancellationToken = default) diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionExtensionsTests.cs index 4920817b664..620470b9fb4 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionExtensionsTests.cs @@ -13,7 +13,7 @@ public class RealtimeSessionExtensionsTests [Fact] public void GetService_NullSession_Throws() { - Assert.Throws("session", () => ((IRealtimeSession)null!).GetService()); + Assert.Throws("session", () => ((IRealtimeClientSession)null!).GetService()); } [Fact] @@ -44,7 +44,7 @@ public void GetService_WithServiceKey_ReturnsNull() public void GetService_ReturnsInterfaceType() { using var session = new TestRealtimeSession(); - var result = session.GetService(); + var result = session.GetService(); Assert.Same(session, result); } } From 6d50fece4185748b15bc76c3f452e64e56bacff6 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Wed, 4 Mar 2026 14:48:30 -0800 Subject: [PATCH 53/92] Fix null DefaultConversationConfiguration when ExcludeFromConversation is unset --- .../OpenAIRealtimeSession.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 9abe60079be..fe5d16ac4e1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -240,12 +240,12 @@ private async Task SendResponseCreateAsync(RealtimeClientResponseCreateMessage r } // Conversation mode. - responseOptions.DefaultConversationConfiguration = responseCreate.ExcludeFromConversation switch + if (responseCreate.ExcludeFromConversation is bool excludeFromConversation) { - true => Sdk.RealtimeResponseDefaultConversationConfiguration.None, - false => Sdk.RealtimeResponseDefaultConversationConfiguration.Auto, - _ => null, - }; + responseOptions.DefaultConversationConfiguration = excludeFromConversation + ? Sdk.RealtimeResponseDefaultConversationConfiguration.None + : Sdk.RealtimeResponseDefaultConversationConfiguration.Auto; + } // Input items. if (responseCreate.Items is { } items) From 7269c159f31499997c60a61909cb6de94a2fcbc9 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Wed, 4 Mar 2026 16:52:01 -0800 Subject: [PATCH 54/92] Remove IDisposable from IRealtimeClientSession, keep only IAsyncDisposable --- .../Realtime/DelegatingRealtimeSession.cs | 26 +--- .../Realtime/IRealtimeClientSession.cs | 2 +- .../OpenAIRealtimeSession.cs | 11 -- .../Realtime/OpenTelemetryRealtimeSession.cs | 12 +- .../TestRealtimeSession.cs | 6 - .../OpenAIRealtimeSessionTests.cs | 26 ++-- .../DelegatingRealtimeSessionTests.cs | 58 ++++---- .../FunctionInvokingRealtimeSessionTests.cs | 82 +++++------ .../Realtime/LoggingRealtimeSessionTests.cs | 68 ++++----- .../OpenTelemetryRealtimeSessionTests.cs | 130 +++++++++--------- .../Realtime/RealtimeSessionBuilderTests.cs | 42 +++--- .../RealtimeSessionExtensionsTests.cs | 17 +-- 12 files changed, 217 insertions(+), 263 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs index 6267503fe99..aeeb46a04a8 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs @@ -31,13 +31,6 @@ protected DelegatingRealtimeSession(IRealtimeClientSession innerSession) InnerSession = Throw.IfNull(innerSession); } - /// - public void Dispose() - { - Dispose(disposing: true); - GC.SuppressFinalize(this); - } - /// public async ValueTask DisposeAsync() { @@ -51,14 +44,7 @@ public async ValueTask DisposeAsync() protected virtual async ValueTask DisposeAsyncCore() #pragma warning restore EA0014 { - if (InnerSession is IAsyncDisposable asyncDisposable) - { - await asyncDisposable.DisposeAsync().ConfigureAwait(false); - } - else - { - InnerSession.Dispose(); - } + await InnerSession.DisposeAsync().ConfigureAwait(false); } /// Gets the inner . @@ -90,14 +76,4 @@ public virtual IAsyncEnumerable GetStreamingResponseAsync serviceKey is null && serviceType.IsInstanceOfType(this) ? this : InnerSession.GetService(serviceType, serviceKey); } - - /// Provides a mechanism for releasing unmanaged resources. - /// if being called from ; otherwise, . - protected virtual void Dispose(bool disposing) - { - if (disposing) - { - InnerSession.Dispose(); - } - } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClientSession.cs index ef1b3e26bc9..1632a0c91f5 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClientSession.cs @@ -13,7 +13,7 @@ namespace Microsoft.Extensions.AI; /// Represents a real-time session. /// This interface provides methods to manage a real-time session and to interact with the real-time model. [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public interface IRealtimeClientSession : IDisposable, IAsyncDisposable +public interface IRealtimeClientSession : IAsyncDisposable { /// Updates the session with new options. /// The new session options. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index fe5d16ac4e1..1363ae85142 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -195,17 +195,6 @@ public async IAsyncEnumerable GetStreamingResponseAsync( null; } - /// - public void Dispose() - { - if (Interlocked.Exchange(ref _disposed, 1) != 0) - { - return; - } - - _sessionClient?.Dispose(); - } - /// public ValueTask DisposeAsync() { diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs index 1f40b805c45..6745febdd01 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs @@ -140,15 +140,11 @@ public JsonSerializerOptions JsonSerializerOptions } /// - protected override void Dispose(bool disposing) + protected override async ValueTask DisposeAsyncCore() { - if (disposing) - { - _activitySource.Dispose(); - _meter.Dispose(); - } - - base.Dispose(disposing); + _activitySource.Dispose(); + _meter.Dispose(); + await base.DisposeAsyncCore().ConfigureAwait(false); } /// diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs index 69cb64a9bd8..3a2ae3acec8 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs @@ -56,12 +56,6 @@ public IAsyncEnumerable GetStreamingResponseAsync( return serviceKey is null && serviceType.IsInstanceOfType(this) ? this : null; } - /// - public void Dispose() - { - // No-op for test implementation - } - /// public ValueTask DisposeAsync() { diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs index 2418f682319..3730a03f2f2 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs @@ -13,9 +13,9 @@ namespace Microsoft.Extensions.AI; public class OpenAIRealtimeSessionTests { [Fact] - public void GetService_ReturnsExpectedServices() + public async Task GetService_ReturnsExpectedServices() { - using IRealtimeClientSession session = new OpenAIRealtimeSession("key", "model"); + await using IRealtimeClientSession session = new OpenAIRealtimeSession("key", "model"); Assert.Same(session, session.GetService(typeof(OpenAIRealtimeSession))); Assert.Same(session, session.GetService(typeof(IRealtimeClientSession))); @@ -24,48 +24,48 @@ public void GetService_ReturnsExpectedServices() } [Fact] - public void GetService_NullServiceType_Throws() + public async Task GetService_NullServiceType_Throws() { - using IRealtimeClientSession session = new OpenAIRealtimeSession("key", "model"); + await using IRealtimeClientSession session = new OpenAIRealtimeSession("key", "model"); Assert.Throws("serviceType", () => session.GetService(null!)); } [Fact] - public void Dispose_CanBeCalledMultipleTimes() + public async Task DisposeAsync_CanBeCalledMultipleTimes() { IRealtimeClientSession session = new OpenAIRealtimeSession("key", "model"); - session.Dispose(); + await session.DisposeAsync(); // Second dispose should not throw. - session.Dispose(); + await session.DisposeAsync(); Assert.Null(session.GetService(typeof(string))); } [Fact] - public void Options_InitiallyNull() + public async Task Options_InitiallyNull() { - using var session = new OpenAIRealtimeSession("key", "model"); + await using var session = new OpenAIRealtimeSession("key", "model"); Assert.Null(session.Options); } [Fact] public async Task UpdateAsync_NullOptions_Throws() { - using var session = new OpenAIRealtimeSession("key", "model"); + await using var session = new OpenAIRealtimeSession("key", "model"); await Assert.ThrowsAsync("options", () => session.UpdateAsync(null!)); } [Fact] public async Task SendAsync_NullMessage_Throws() { - using var session = new OpenAIRealtimeSession("key", "model"); + await using var session = new OpenAIRealtimeSession("key", "model"); await Assert.ThrowsAsync("message", () => session.SendAsync(null!)); } [Fact] public async Task SendAsync_CancelledToken_ReturnsSilently() { - using var session = new OpenAIRealtimeSession("key", "model"); + await using var session = new OpenAIRealtimeSession("key", "model"); using var cts = new CancellationTokenSource(); cts.Cancel(); @@ -77,7 +77,7 @@ public async Task SendAsync_CancelledToken_ReturnsSilently() [Fact] public async Task ConnectAsync_CancelledToken_Throws() { - using var session = new OpenAIRealtimeSession("key", "model"); + await using var session = new OpenAIRealtimeSession("key", "model"); using var cts = new CancellationTokenSource(); cts.Cancel(); diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs index 04e80e22023..19c6298e592 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs @@ -21,11 +21,11 @@ public void Ctor_NullInnerSession_Throws() } [Fact] - public void Options_DelegatesToInner() + public async Task Options_DelegatesToInner() { var expectedOptions = new RealtimeSessionOptions { Model = "test-model" }; - using var inner = new TestRealtimeSession { Options = expectedOptions }; - using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var inner = new TestRealtimeSession { Options = expectedOptions }; + await using var delegating = new NoOpDelegatingRealtimeSession(inner); Assert.Same(expectedOptions, delegating.Options); } @@ -35,7 +35,7 @@ public async Task UpdateAsync_DelegatesToInner() { var called = false; var sentOptions = new RealtimeSessionOptions { Instructions = "Be helpful" }; - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { UpdateAsyncCallback = (options, _) => { @@ -44,7 +44,7 @@ public async Task UpdateAsync_DelegatesToInner() return Task.CompletedTask; }, }; - using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var delegating = new NoOpDelegatingRealtimeSession(inner); await delegating.UpdateAsync(sentOptions); Assert.True(called); @@ -55,7 +55,7 @@ public async Task SendAsync_DelegatesToInner() { var called = false; var sentMessage = new RealtimeClientMessage { MessageId = "evt_001" }; - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { SendAsyncCallback = (msg, _) => { @@ -64,7 +64,7 @@ public async Task SendAsync_DelegatesToInner() return Task.CompletedTask; }, }; - using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var delegating = new NoOpDelegatingRealtimeSession(inner); await delegating.SendAsync(sentMessage); Assert.True(called); @@ -74,11 +74,11 @@ public async Task SendAsync_DelegatesToInner() public async Task GetStreamingResponseAsync_DelegatesToInner() { var expected = new RealtimeServerMessage { Type = RealtimeServerMessageType.Error, MessageId = "evt_002" }; - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { GetStreamingResponseAsyncCallback = (ct) => YieldSingle(expected, ct), }; - using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var delegating = new NoOpDelegatingRealtimeSession(inner); var messages = new List(); await foreach (var msg in delegating.GetStreamingResponseAsync()) @@ -91,10 +91,10 @@ public async Task GetStreamingResponseAsync_DelegatesToInner() } [Fact] - public void GetService_ReturnsSelfForMatchingType() + public async Task GetService_ReturnsSelfForMatchingType() { - using var inner = new TestRealtimeSession(); - using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var inner = new TestRealtimeSession(); + await using var delegating = new NoOpDelegatingRealtimeSession(inner); Assert.Same(delegating, delegating.GetService(typeof(NoOpDelegatingRealtimeSession))); Assert.Same(delegating, delegating.GetService(typeof(DelegatingRealtimeSession))); @@ -102,10 +102,10 @@ public void GetService_ReturnsSelfForMatchingType() } [Fact] - public void GetService_DelegatesToInnerForUnknownType() + public async Task GetService_DelegatesToInnerForUnknownType() { - using var inner = new TestRealtimeSession(); - using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var inner = new TestRealtimeSession(); + await using var delegating = new NoOpDelegatingRealtimeSession(inner); // TestRealtimeSession returns itself for matching types Assert.Same(inner, delegating.GetService(typeof(TestRealtimeSession))); @@ -113,32 +113,32 @@ public void GetService_DelegatesToInnerForUnknownType() } [Fact] - public void GetService_WithServiceKey_DelegatesToInner() + public async Task GetService_WithServiceKey_DelegatesToInner() { - using var inner = new TestRealtimeSession(); - using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var inner = new TestRealtimeSession(); + await using var delegating = new NoOpDelegatingRealtimeSession(inner); // With a non-null key, delegating should NOT return itself even for matching types Assert.Null(delegating.GetService(typeof(NoOpDelegatingRealtimeSession), "someKey")); } [Fact] - public void GetService_NullServiceType_Throws() + public async Task GetService_NullServiceType_Throws() { - using var inner = new TestRealtimeSession(); - using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var inner = new TestRealtimeSession(); + await using var delegating = new NoOpDelegatingRealtimeSession(inner); Assert.Throws("serviceType", () => delegating.GetService(null!)); } [Fact] - public void Dispose_DisposesInner() + public async Task DisposeAsync_DisposesInner() { var disposed = false; - using var inner = new DisposableTestRealtimeSession(() => disposed = true); + await using var inner = new DisposableTestRealtimeSession(() => disposed = true); var delegating = new NoOpDelegatingRealtimeSession(inner); - delegating.Dispose(); + await delegating.DisposeAsync(); Assert.True(disposed); } @@ -149,7 +149,7 @@ public async Task UpdateAsync_FlowsCancellationToken() using var cts = new CancellationTokenSource(); var sentOptions = new RealtimeSessionOptions(); - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { UpdateAsyncCallback = (options, ct) => { @@ -157,7 +157,7 @@ public async Task UpdateAsync_FlowsCancellationToken() return Task.CompletedTask; }, }; - using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var delegating = new NoOpDelegatingRealtimeSession(inner); await delegating.UpdateAsync(sentOptions, cts.Token); Assert.Equal(cts.Token, capturedToken); @@ -170,7 +170,7 @@ public async Task SendAsync_FlowsCancellationToken() using var cts = new CancellationTokenSource(); var sentMessage = new RealtimeClientMessage(); - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { SendAsyncCallback = (msg, ct) => { @@ -178,7 +178,7 @@ public async Task SendAsync_FlowsCancellationToken() return Task.CompletedTask; }, }; - using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var delegating = new NoOpDelegatingRealtimeSession(inner); await delegating.SendAsync(sentMessage, cts.Token); Assert.Equal(cts.Token, capturedToken); @@ -224,8 +224,6 @@ public IAsyncEnumerable GetStreamingResponseAsync( public object? GetService(Type serviceType, object? serviceKey = null) => null; - public void Dispose() => _onDispose(); - public ValueTask DisposeAsync() { _onDispose(); diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs index 74cc46c4866..d751c8067d4 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs @@ -24,10 +24,10 @@ public void Ctor_NullInnerSession_Throws() } [Fact] - public void Properties_DefaultValues() + public async Task Properties_DefaultValues() { - using var inner = new TestRealtimeSession(); - using var session = new FunctionInvokingRealtimeSession(inner); + await using var inner = new TestRealtimeSession(); + await using var session = new FunctionInvokingRealtimeSession(inner); Assert.False(session.IncludeDetailedErrors); Assert.False(session.AllowConcurrentInvocation); @@ -39,20 +39,20 @@ public void Properties_DefaultValues() } [Fact] - public void MaximumIterationsPerRequest_InvalidValue_Throws() + public async Task MaximumIterationsPerRequest_InvalidValue_Throws() { - using var inner = new TestRealtimeSession(); - using var session = new FunctionInvokingRealtimeSession(inner); + await using var inner = new TestRealtimeSession(); + await using var session = new FunctionInvokingRealtimeSession(inner); Assert.Throws("value", () => session.MaximumIterationsPerRequest = 0); Assert.Throws("value", () => session.MaximumIterationsPerRequest = -1); } [Fact] - public void MaximumConsecutiveErrorsPerRequest_InvalidValue_Throws() + public async Task MaximumConsecutiveErrorsPerRequest_InvalidValue_Throws() { - using var inner = new TestRealtimeSession(); - using var session = new FunctionInvokingRealtimeSession(inner); + await using var inner = new TestRealtimeSession(); + await using var session = new FunctionInvokingRealtimeSession(inner); Assert.Throws("value", () => session.MaximumConsecutiveErrorsPerRequest = -1); @@ -70,11 +70,11 @@ public async Task GetStreamingResponseAsync_NoFunctionCalls_PassesThrough() new() { Type = RealtimeServerMessageType.ResponseDone, MessageId = "evt_002" }, }; - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { GetStreamingResponseAsyncCallback = (ct) => YieldMessages(serverMessages, ct), }; - using var session = new FunctionInvokingRealtimeSession(inner); + await using var session = new FunctionInvokingRealtimeSession(inner); var received = new List(); await foreach (var msg in session.GetStreamingResponseAsync()) @@ -96,7 +96,7 @@ public async Task GetStreamingResponseAsync_FunctionCall_InvokesAndInjectsResult "Gets the weather"); var injectedMessages = new List(); - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [getWeather] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages( @@ -110,7 +110,7 @@ public async Task GetStreamingResponseAsync_FunctionCall_InvokesAndInjectsResult }, }; - using var session = new FunctionInvokingRealtimeSession(inner); + await using var session = new FunctionInvokingRealtimeSession(inner); var received = new List(); await foreach (var msg in session.GetStreamingResponseAsync()) @@ -145,7 +145,7 @@ public async Task GetStreamingResponseAsync_FunctionCall_FromAdditionalTools() "Gets weather"); var injectedMessages = new List(); - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ @@ -158,7 +158,7 @@ public async Task GetStreamingResponseAsync_FunctionCall_FromAdditionalTools() }, }; - using var session = new FunctionInvokingRealtimeSession(inner) + await using var session = new FunctionInvokingRealtimeSession(inner) { AdditionalTools = [getWeather], }; @@ -190,14 +190,14 @@ public async Task GetStreamingResponseAsync_MaxIterations_StopsInvoking() var messages = Enumerable.Range(0, 5).Select(i => CreateFunctionCallOutputItemMessage($"call_{i}", "counter", null)).ToList(); - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [countFunc] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages(messages, ct), SendAsyncCallback = (_, _) => Task.CompletedTask, }; - using var session = new FunctionInvokingRealtimeSession(inner) + await using var session = new FunctionInvokingRealtimeSession(inner) { MaximumIterationsPerRequest = 2, }; @@ -224,7 +224,7 @@ public async Task GetStreamingResponseAsync_FunctionInvoker_CustomDelegate() "my_func", "Test"); - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [myFunc] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages( @@ -234,7 +234,7 @@ public async Task GetStreamingResponseAsync_FunctionInvoker_CustomDelegate() SendAsyncCallback = (_, _) => Task.CompletedTask, }; - using var session = new FunctionInvokingRealtimeSession(inner) + await using var session = new FunctionInvokingRealtimeSession(inner) { FunctionInvoker = (context, ct) => { @@ -255,7 +255,7 @@ public async Task GetStreamingResponseAsync_FunctionInvoker_CustomDelegate() public async Task GetStreamingResponseAsync_UnknownFunction_SendsErrorByDefault() { var injectedMessages = new List(); - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ @@ -268,7 +268,7 @@ public async Task GetStreamingResponseAsync_UnknownFunction_SendsErrorByDefault( }, }; - using var session = new FunctionInvokingRealtimeSession(inner); + await using var session = new FunctionInvokingRealtimeSession(inner); await foreach (var msg in session.GetStreamingResponseAsync()) { @@ -291,7 +291,7 @@ public async Task GetStreamingResponseAsync_FunctionError_IncludesDetailedErrors "Fails"); var injectedMessages = new List(); - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [failFunc] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages( @@ -305,7 +305,7 @@ public async Task GetStreamingResponseAsync_FunctionError_IncludesDetailedErrors }, }; - using var session = new FunctionInvokingRealtimeSession(inner) + await using var session = new FunctionInvokingRealtimeSession(inner) { IncludeDetailedErrors = true, }; @@ -330,7 +330,7 @@ public async Task GetStreamingResponseAsync_FunctionError_HidesDetailsWhenNotEna "Fails"); var injectedMessages = new List(); - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [failFunc] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages( @@ -344,7 +344,7 @@ public async Task GetStreamingResponseAsync_FunctionError_HidesDetailsWhenNotEna }, }; - using var session = new FunctionInvokingRealtimeSession(inner) + await using var session = new FunctionInvokingRealtimeSession(inner) { IncludeDetailedErrors = false, }; @@ -361,10 +361,10 @@ public async Task GetStreamingResponseAsync_FunctionError_HidesDetailsWhenNotEna } [Fact] - public void GetService_ReturnsSelf() + public async Task GetService_ReturnsSelf() { - using var inner = new TestRealtimeSession(); - using var session = new FunctionInvokingRealtimeSession(inner); + await using var inner = new TestRealtimeSession(); + await using var session = new FunctionInvokingRealtimeSession(inner); Assert.Same(session, session.GetService(typeof(FunctionInvokingRealtimeSession))); Assert.Same(session, session.GetService(typeof(IRealtimeClientSession))); @@ -375,7 +375,7 @@ public void GetService_ReturnsSelf() public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_StopsLoop() { var injectedMessages = new List(); - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ @@ -389,7 +389,7 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_StopsLoop() }, }; - using var session = new FunctionInvokingRealtimeSession(inner) + await using var session = new FunctionInvokingRealtimeSession(inner) { TerminateOnUnknownCalls = true, }; @@ -411,7 +411,7 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_StopsLoop() public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_False_SendsError() { var injectedMessages = new List(); - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ @@ -424,7 +424,7 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_False_SendsE }, }; - using var session = new FunctionInvokingRealtimeSession(inner) + await using var session = new FunctionInvokingRealtimeSession(inner) { TerminateOnUnknownCalls = false, }; @@ -495,14 +495,14 @@ public async Task GetStreamingResponseAsync_ConcurrentInvocation_InvokesInParall Item = combinedItem, }; - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [slowFunc] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages([combinedMessage], ct), SendAsyncCallback = (_, _) => Task.CompletedTask, }; - using var session = new FunctionInvokingRealtimeSession(inner) + await using var session = new FunctionInvokingRealtimeSession(inner) { AllowConcurrentInvocation = true, }; @@ -538,14 +538,14 @@ public async Task GetStreamingResponseAsync_ConsecutiveErrors_ExceedsLimit_Throw CreateFunctionCallOutputItemMessage("call_4", "fail_func", null), }; - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [failFunc] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages(messages, ct), SendAsyncCallback = (_, _) => Task.CompletedTask, }; - using var session = new FunctionInvokingRealtimeSession(inner) + await using var session = new FunctionInvokingRealtimeSession(inner) { MaximumConsecutiveErrorsPerRequest = 1, }; @@ -568,9 +568,9 @@ public void UseFunctionInvocation_NullBuilder_Throws() } [Fact] - public void UseFunctionInvocation_ConfigureCallback_IsInvoked() + public async Task UseFunctionInvocation_ConfigureCallback_IsInvoked() { - using var inner = new TestRealtimeSession(); + await using var inner = new TestRealtimeSession(); var builder = new RealtimeSessionBuilder(inner); bool configured = false; @@ -581,7 +581,7 @@ public void UseFunctionInvocation_ConfigureCallback_IsInvoked() session.MaximumIterationsPerRequest = 10; }); - using var pipeline = builder.Build(); + await using var pipeline = builder.Build(); Assert.True(configured); var funcSession = pipeline.GetService(typeof(FunctionInvokingRealtimeSession)); @@ -600,7 +600,7 @@ public async Task GetStreamingResponseAsync_NonInvocableTool_TerminatesLoop() var declaration = AIFunctionFactory.CreateDeclaration("my_declaration", "A non-invocable tool", schema); var injectedMessages = new List(); - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { Options = new RealtimeSessionOptions { Tools = [declaration] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages( @@ -615,7 +615,7 @@ public async Task GetStreamingResponseAsync_NonInvocableTool_TerminatesLoop() }, }; - using var session = new FunctionInvokingRealtimeSession(inner); + await using var session = new FunctionInvokingRealtimeSession(inner); var received = new List(); await foreach (var msg in session.GetStreamingResponseAsync()) diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs index 0a866631ee4..8f6b662cb12 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs @@ -17,17 +17,17 @@ namespace Microsoft.Extensions.AI; public class LoggingRealtimeSessionTests { [Fact] - public void LoggingRealtimeSession_InvalidArgs_Throws() + public async Task LoggingRealtimeSession_InvalidArgs_Throws() { - using var innerSession = new TestRealtimeSession(); + await using var innerSession = new TestRealtimeSession(); Assert.Throws("innerSession", () => new LoggingRealtimeSession(null!, NullLogger.Instance)); Assert.Throws("logger", () => new LoggingRealtimeSession(innerSession, null!)); } [Fact] - public void UseLogging_AvoidsInjectingNopSession() + public async Task UseLogging_AvoidsInjectingNopSession() { - using var innerSession = new TestRealtimeSession(); + await using var innerSession = new TestRealtimeSession(); Assert.Null(innerSession.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build().GetService(typeof(LoggingRealtimeSession))); Assert.Same(innerSession, innerSession.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build().GetService(typeof(IRealtimeClientSession))); @@ -55,12 +55,12 @@ public async Task UpdateAsync_LogsInvocationAndCompletion(LogLevel level) c.AddLogging(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(level)); var services = c.BuildServiceProvider(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { UpdateAsyncCallback = (options, cancellationToken) => Task.CompletedTask, }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseLogging() .Build(services); @@ -98,12 +98,12 @@ public async Task SendAsync_LogsInvocationAndCompletion(LogLevel level) c.AddLogging(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(level)); var services = c.BuildServiceProvider(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { SendAsyncCallback = (message, cancellationToken) => Task.CompletedTask, }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseLogging() .Build(services); @@ -138,7 +138,7 @@ public async Task GetStreamingResponseAsync_LogsMessagesReceived(LogLevel level) var collector = new FakeLogCollector(); using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(level)); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { GetStreamingResponseAsyncCallback = (cancellationToken) => GetMessagesAsync() }; @@ -150,7 +150,7 @@ static async IAsyncEnumerable GetMessagesAsync() yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.OutputAudioDelta, MessageId = "event-2" }; } - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseLogging(loggerFactory) .Build(); @@ -191,7 +191,7 @@ public async Task UpdateAsync_LogsCancellation() using var cts = new CancellationTokenSource(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { UpdateAsyncCallback = (options, cancellationToken) => { @@ -199,7 +199,7 @@ public async Task UpdateAsync_LogsCancellation() }, }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseLogging(loggerFactory) .Build(); @@ -219,7 +219,7 @@ public async Task UpdateAsync_LogsErrors() var collector = new FakeLogCollector(); using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { UpdateAsyncCallback = (options, cancellationToken) => { @@ -227,7 +227,7 @@ public async Task UpdateAsync_LogsErrors() }, }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseLogging(loggerFactory) .Build(); @@ -248,7 +248,7 @@ public async Task GetStreamingResponseAsync_LogsCancellation() using var cts = new CancellationTokenSource(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { GetStreamingResponseAsyncCallback = (cancellationToken) => ThrowCancellationAsync(cancellationToken) }; @@ -262,7 +262,7 @@ static async IAsyncEnumerable ThrowCancellationAsync([Enu #pragma warning restore CS0162 } - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseLogging(loggerFactory) .Build(); @@ -288,7 +288,7 @@ public async Task GetStreamingResponseAsync_LogsErrors() var collector = new FakeLogCollector(); using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { GetStreamingResponseAsyncCallback = (cancellationToken) => ThrowErrorAsync() }; @@ -302,7 +302,7 @@ static async IAsyncEnumerable ThrowErrorAsync() #pragma warning restore CS0162 } - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseLogging(loggerFactory) .Build(); @@ -322,13 +322,13 @@ await Assert.ThrowsAsync(async () => } [Fact] - public void GetService_ReturnsLoggingSessionWhenRequested() + public async Task GetService_ReturnsLoggingSessionWhenRequested() { using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddFakeLogging()); - using var innerSession = new TestRealtimeSession(); + await using var innerSession = new TestRealtimeSession(); - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseLogging(loggerFactory) .Build(); @@ -345,7 +345,7 @@ public async Task SendAsync_LogsCancellation() using var cts = new CancellationTokenSource(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { SendAsyncCallback = (message, cancellationToken) => { @@ -353,7 +353,7 @@ public async Task SendAsync_LogsCancellation() }, }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseLogging(loggerFactory) .Build(); @@ -374,7 +374,7 @@ public async Task SendAsync_LogsErrors() var collector = new FakeLogCollector(); using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { SendAsyncCallback = (message, cancellationToken) => { @@ -382,7 +382,7 @@ public async Task SendAsync_LogsErrors() }, }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseLogging(loggerFactory) .Build(); @@ -397,19 +397,19 @@ await Assert.ThrowsAsync(() => } [Fact] - public void JsonSerializerOptions_NullValue_Throws() + public async Task JsonSerializerOptions_NullValue_Throws() { - using var innerSession = new TestRealtimeSession(); - using var session = new LoggingRealtimeSession(innerSession, NullLogger.Instance); + await using var innerSession = new TestRealtimeSession(); + await using var session = new LoggingRealtimeSession(innerSession, NullLogger.Instance); Assert.Throws("value", () => session.JsonSerializerOptions = null!); } [Fact] - public void JsonSerializerOptions_Roundtrip() + public async Task JsonSerializerOptions_Roundtrip() { - using var innerSession = new TestRealtimeSession(); - using var session = new LoggingRealtimeSession(innerSession, NullLogger.Instance); + await using var innerSession = new TestRealtimeSession(); + await using var session = new LoggingRealtimeSession(innerSession, NullLogger.Instance); var customOptions = new System.Text.Json.JsonSerializerOptions(); session.JsonSerializerOptions = customOptions; @@ -425,13 +425,13 @@ public void UseLogging_NullBuilder_Throws() } [Fact] - public void UseLogging_ConfigureCallback_IsInvoked() + public async Task UseLogging_ConfigureCallback_IsInvoked() { - using var innerSession = new TestRealtimeSession(); + await using var innerSession = new TestRealtimeSession(); using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddFakeLogging()); bool configured = false; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseLogging(loggerFactory, configure: s => { diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs index 7b6023e2e03..211c84dabd7 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs @@ -28,7 +28,7 @@ public async Task ExpectedInformationLogged_GetStreamingResponseAsync(bool enabl .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { @@ -76,7 +76,7 @@ static async IAsyncEnumerable CallbackAsync([EnumeratorCa }; } - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName, configure: instance => { @@ -194,7 +194,7 @@ public async Task UpdateAsync_TracesOperation() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { UpdateAsyncCallback = (options, cancellationToken) => Task.CompletedTask, GetServiceCallback = (serviceType, serviceKey) => @@ -202,7 +202,7 @@ public async Task UpdateAsync_TracesOperation() null, }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName, configure: instance => { @@ -242,12 +242,12 @@ public async Task UpdateAsync_TracesError() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { UpdateAsyncCallback = (options, cancellationToken) => throw new InvalidOperationException("Test error"), }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); @@ -270,7 +270,7 @@ public async Task GetStreamingResponseAsync_TracesError() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetStreamingResponseAsyncCallback = (cancellationToken) => ThrowingCallbackAsync(cancellationToken), @@ -284,7 +284,7 @@ static async IAsyncEnumerable ThrowingCallbackAsync([Enum throw new InvalidOperationException("Streaming error"); } - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); @@ -313,7 +313,7 @@ public async Task GetStreamingResponseAsync_TracesErrorFromResponse() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetStreamingResponseAsyncCallback = (cancellationToken) => ErrorResponseCallbackAsync(cancellationToken), @@ -332,7 +332,7 @@ static async IAsyncEnumerable ErrorResponseCallbackAsync( }; } - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); @@ -363,7 +363,7 @@ public async Task DefaultVoiceSpeed_NotLogged() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { @@ -381,7 +381,7 @@ static async IAsyncEnumerable EmptyCallbackAsync([Enumera yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); } - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); @@ -409,7 +409,7 @@ public async Task NoListeners_NoActivityCreated() .AddSource("different-source") .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetStreamingResponseAsyncCallback = (cancellationToken) => EmptyCallbackAsync(cancellationToken), @@ -426,7 +426,7 @@ static async IAsyncEnumerable EmptyCallbackAsync([Enumera } var sourceName = Guid.NewGuid().ToString(); - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); @@ -443,9 +443,9 @@ static async IAsyncEnumerable EmptyCallbackAsync([Enumera } [Fact] - public void InvalidArgs_Throws() + public async Task InvalidArgs_Throws() { - using var innerSession = new TestRealtimeSession(); + await using var innerSession = new TestRealtimeSession(); Assert.Throws("innerSession", () => new OpenTelemetryRealtimeSession(null!)); Assert.Throws("value", () => new OpenTelemetryRealtimeSession(innerSession).JsonSerializerOptions = null!); @@ -454,17 +454,17 @@ public void InvalidArgs_Throws() [Fact] public async Task UpdateAsync_InvalidArgs_Throws() { - using var innerSession = new TestRealtimeSession(); - using var session = new OpenTelemetryRealtimeSession(innerSession); + await using var innerSession = new TestRealtimeSession(); + await using var session = new OpenTelemetryRealtimeSession(innerSession); await Assert.ThrowsAsync("options", () => session.UpdateAsync(null!)); } [Fact] - public void GetService_ReturnsActivitySource() + public async Task GetService_ReturnsActivitySource() { - using var innerSession = new TestRealtimeSession(); - using var session = new OpenTelemetryRealtimeSession(innerSession); + await using var innerSession = new TestRealtimeSession(); + await using var session = new OpenTelemetryRealtimeSession(innerSession); var activitySource = session.GetService(typeof(ActivitySource)); Assert.NotNull(activitySource); @@ -472,10 +472,10 @@ public void GetService_ReturnsActivitySource() } [Fact] - public void GetService_ReturnsSelf() + public async Task GetService_ReturnsSelf() { - using var innerSession = new TestRealtimeSession(); - using var session = new OpenTelemetryRealtimeSession(innerSession); + await using var innerSession = new TestRealtimeSession(); + await using var session = new OpenTelemetryRealtimeSession(innerSession); var self = session.GetService(typeof(OpenTelemetryRealtimeSession)); Assert.Same(session, self); @@ -494,7 +494,7 @@ public async Task TranscriptionSessionKind_Logged() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { @@ -516,7 +516,7 @@ static async IAsyncEnumerable TranscriptionCallbackAsync( yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); } - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); @@ -556,7 +556,7 @@ public async Task ToolChoiceMode_Logged(string modeKey, string expectedValue) .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { @@ -567,7 +567,7 @@ public async Task ToolChoiceMode_Logged(string modeKey, string expectedValue) GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); @@ -596,7 +596,7 @@ public async Task AIFunction_ForcedTool_Logged() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { @@ -606,7 +606,7 @@ public async Task AIFunction_ForcedTool_Logged() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); @@ -636,7 +636,7 @@ public async Task RequireAny_ToolMode_Logged() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { @@ -646,7 +646,7 @@ public async Task RequireAny_ToolMode_Logged() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); @@ -676,7 +676,7 @@ public async Task NoToolChoice_NotLogged() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { @@ -685,7 +685,7 @@ public async Task NoToolChoice_NotLogged() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); @@ -714,7 +714,7 @@ public async Task ToolCallContentInClientMessages_LoggedAsInputMessages() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -722,7 +722,7 @@ public async Task ToolCallContentInClientMessages_LoggedAsInputMessages() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); @@ -757,7 +757,7 @@ public async Task ToolCallContentInServerMessages_LoggedAsOutputMessages() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -765,7 +765,7 @@ public async Task ToolCallContentInServerMessages_LoggedAsOutputMessages() GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithToolCallAsync(cancellationToken), }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); @@ -800,7 +800,7 @@ public async Task ToolContentNotLoggedWithoutSensitiveData() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -808,7 +808,7 @@ public async Task ToolContentNotLoggedWithoutSensitiveData() GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithToolCallAsync(cancellationToken), }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = false) .Build(); @@ -885,7 +885,7 @@ public async Task AudioBufferAppendMessage_LoggedAsInputMessage() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -893,7 +893,7 @@ public async Task AudioBufferAppendMessage_LoggedAsInputMessage() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); @@ -928,7 +928,7 @@ public async Task AudioBufferCommitMessage_LoggedAsInputMessage() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -936,7 +936,7 @@ public async Task AudioBufferCommitMessage_LoggedAsInputMessage() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); @@ -971,7 +971,7 @@ public async Task ResponseCreateMessageWithInstructions_LoggedAsInputMessage() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -979,7 +979,7 @@ public async Task ResponseCreateMessageWithInstructions_LoggedAsInputMessage() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); @@ -1013,7 +1013,7 @@ public async Task ResponseCreateMessageWithItems_LoggedAsInputMessage() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -1021,7 +1021,7 @@ public async Task ResponseCreateMessageWithItems_LoggedAsInputMessage() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); @@ -1055,7 +1055,7 @@ public async Task OutputTextAudioMessage_LoggedAsOutputMessage() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -1063,7 +1063,7 @@ public async Task OutputTextAudioMessage_LoggedAsOutputMessage() GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithTextOutputAsync(cancellationToken), }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); @@ -1096,7 +1096,7 @@ public async Task InputAudioTranscriptionMessage_LoggedAsOutputMessage() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -1104,7 +1104,7 @@ public async Task InputAudioTranscriptionMessage_LoggedAsOutputMessage() GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithTranscriptionAsync(cancellationToken), }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); @@ -1137,7 +1137,7 @@ public async Task ServerErrorMessage_LoggedAsOutputMessage() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -1145,7 +1145,7 @@ public async Task ServerErrorMessage_LoggedAsOutputMessage() GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithServerErrorAsync(cancellationToken), }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); @@ -1178,7 +1178,7 @@ public async Task ConversationItemCreateWithTextContent_LoggedAsInputMessage() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -1186,7 +1186,7 @@ public async Task ConversationItemCreateWithTextContent_LoggedAsInputMessage() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); @@ -1219,7 +1219,7 @@ public async Task DataContentInClientMessage_LoggedWithModality() .AddInMemoryExporter(activities) .Build(); - using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -1227,7 +1227,7 @@ public async Task DataContentInClientMessage_LoggedWithModality() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - using var session = innerSession + await using var session = innerSession .AsBuilder() .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) .Build(); @@ -1335,9 +1335,9 @@ public void UseOpenTelemetry_NullBuilder_Throws() } [Fact] - public void UseOpenTelemetry_ConfigureCallback_IsInvoked() + public async Task UseOpenTelemetry_ConfigureCallback_IsInvoked() { - using var innerSession = new TestRealtimeSession(); + await using var innerSession = new TestRealtimeSession(); var builder = new RealtimeSessionBuilder(innerSession); bool configured = false; @@ -1347,7 +1347,7 @@ public void UseOpenTelemetry_ConfigureCallback_IsInvoked() session.EnableSensitiveData = true; }); - using var pipeline = builder.Build(); + await using var pipeline = builder.Build(); Assert.True(configured); var otelSession = pipeline.GetService(typeof(OpenTelemetryRealtimeSession)); @@ -1358,13 +1358,13 @@ public void UseOpenTelemetry_ConfigureCallback_IsInvoked() } [Fact] - public void Dispose_CanBeCalledMultipleTimes() + public async Task DisposeAsync_CanBeCalledMultipleTimes() { - using var innerSession = new TestRealtimeSession(); + await using var innerSession = new TestRealtimeSession(); var session = new OpenTelemetryRealtimeSession(innerSession); - session.Dispose(); - session.Dispose(); + await session.DisposeAsync(); + await session.DisposeAsync(); // Verifying no exception is thrown on double dispose Assert.NotNull(session); diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs index 4caa268c438..bfd3b7ee72a 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs @@ -27,9 +27,9 @@ public void Ctor_NullFactory_Throws() } [Fact] - public void Build_WithNoMiddleware_ReturnsInnerSession() + public async Task Build_WithNoMiddleware_ReturnsInnerSession() { - using var inner = new TestRealtimeSession(); + await using var inner = new TestRealtimeSession(); var builder = new RealtimeSessionBuilder(inner); var result = builder.Build(); @@ -37,9 +37,9 @@ public void Build_WithNoMiddleware_ReturnsInnerSession() } [Fact] - public void Build_WithFactory_UsesFactory() + public async Task Build_WithFactory_UsesFactory() { - using var inner = new TestRealtimeSession(); + await using var inner = new TestRealtimeSession(); var builder = new RealtimeSessionBuilder(_ => inner); var result = builder.Build(); @@ -47,9 +47,9 @@ public void Build_WithFactory_UsesFactory() } [Fact] - public void Use_NullSessionFactory_Throws() + public async Task Use_NullSessionFactory_Throws() { - using var inner = new TestRealtimeSession(); + await using var inner = new TestRealtimeSession(); var builder = new RealtimeSessionBuilder(inner); Assert.Throws("sessionFactory", () => builder.Use((Func)null!)); @@ -57,9 +57,9 @@ public void Use_NullSessionFactory_Throws() } [Fact] - public void Use_StreamingDelegate_NullFunc_Throws() + public async Task Use_StreamingDelegate_NullFunc_Throws() { - using var inner = new TestRealtimeSession(); + await using var inner = new TestRealtimeSession(); var builder = new RealtimeSessionBuilder(inner); Assert.Throws( @@ -68,16 +68,16 @@ public void Use_StreamingDelegate_NullFunc_Throws() } [Fact] - public void Build_PipelineOrder_FirstAddedIsOutermost() + public async Task Build_PipelineOrder_FirstAddedIsOutermost() { var callOrder = new List(); - using var inner = new TestRealtimeSession(); + await using var inner = new TestRealtimeSession(); var builder = new RealtimeSessionBuilder(inner); builder.Use(session => new OrderTrackingSession(session, "first", callOrder)); builder.Use(session => new OrderTrackingSession(session, "second", callOrder)); - using var pipeline = builder.Build(); + await using var pipeline = builder.Build(); // The outermost should be "first" (added first) var outermost = Assert.IsType(pipeline); @@ -90,10 +90,10 @@ public void Build_PipelineOrder_FirstAddedIsOutermost() } [Fact] - public void Build_WithServiceProvider_PassesToFactory() + public async Task Build_WithServiceProvider_PassesToFactory() { IServiceProvider? capturedServices = null; - using var inner = new TestRealtimeSession(); + await using var inner = new TestRealtimeSession(); var builder = new RealtimeSessionBuilder(inner); builder.Use((session, services) => @@ -109,10 +109,10 @@ public void Build_WithServiceProvider_PassesToFactory() } [Fact] - public void Build_NullServiceProvider_UsesEmptyProvider() + public async Task Build_NullServiceProvider_UsesEmptyProvider() { IServiceProvider? capturedServices = null; - using var inner = new TestRealtimeSession(); + await using var inner = new TestRealtimeSession(); var builder = new RealtimeSessionBuilder(inner); builder.Use((session, services) => @@ -127,9 +127,9 @@ public void Build_NullServiceProvider_UsesEmptyProvider() } [Fact] - public void Use_ReturnsSameBuilder_ForChaining() + public async Task Use_ReturnsSameBuilder_ForChaining() { - using var inner = new TestRealtimeSession(); + await using var inner = new TestRealtimeSession(); var builder = new RealtimeSessionBuilder(inner); var returned = builder.Use(s => s); @@ -140,7 +140,7 @@ public void Use_ReturnsSameBuilder_ForChaining() public async Task Use_WithStreamingDelegate_InterceptsStreaming() { var intercepted = false; - using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeSession { GetStreamingResponseAsyncCallback = (ct) => YieldSingle(new RealtimeServerMessage { MessageId = "inner" }, ct), }; @@ -152,7 +152,7 @@ public async Task Use_WithStreamingDelegate_InterceptsStreaming() return innerSession.GetStreamingResponseAsync(ct); }); - using var pipeline = builder.Build(); + await using var pipeline = builder.Build(); await foreach (var msg in pipeline.GetStreamingResponseAsync()) { Assert.Equal("inner", msg.MessageId); @@ -168,9 +168,9 @@ public void AsBuilder_NullSession_Throws() } [Fact] - public void AsBuilder_ReturnsBuilder() + public async Task AsBuilder_ReturnsBuilder() { - using var inner = new TestRealtimeSession(); + await using var inner = new TestRealtimeSession(); var builder = inner.AsBuilder(); Assert.NotNull(builder); diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionExtensionsTests.cs index 620470b9fb4..0e1986ee713 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionExtensionsTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Threading.Tasks; using Xunit; #pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. @@ -17,33 +18,33 @@ public void GetService_NullSession_Throws() } [Fact] - public void GetService_ReturnsMatchingService() + public async Task GetService_ReturnsMatchingService() { - using var session = new TestRealtimeSession(); + await using var session = new TestRealtimeSession(); var result = session.GetService(); Assert.Same(session, result); } [Fact] - public void GetService_ReturnsNullForNonMatchingType() + public async Task GetService_ReturnsNullForNonMatchingType() { - using var session = new TestRealtimeSession(); + await using var session = new TestRealtimeSession(); var result = session.GetService(); Assert.Null(result); } [Fact] - public void GetService_WithServiceKey_ReturnsNull() + public async Task GetService_WithServiceKey_ReturnsNull() { - using var session = new TestRealtimeSession(); + await using var session = new TestRealtimeSession(); var result = session.GetService("someKey"); Assert.Null(result); } [Fact] - public void GetService_ReturnsInterfaceType() + public async Task GetService_ReturnsInterfaceType() { - using var session = new TestRealtimeSession(); + await using var session = new TestRealtimeSession(); var result = session.GetService(); Assert.Same(session, result); } From 8843a12e7e43c16d28c05e6523a35f6c609db17d Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Wed, 4 Mar 2026 17:06:49 -0800 Subject: [PATCH 55/92] Remove ConversationId from RealtimeServerResponseCreatedMessage --- .../Realtime/RealtimeServerResponseCreatedMessage.cs | 10 ---------- .../OpenAIRealtimeSession.cs | 1 - .../Realtime/OpenTelemetryRealtimeSession.cs | 5 ----- .../Realtime/RealtimeServerMessageTests.cs | 3 --- .../Realtime/OpenTelemetryRealtimeSessionTests.cs | 2 -- 5 files changed, 21 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs index d5bc496c4a1..8c946186db1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs @@ -45,16 +45,6 @@ public RealtimeServerResponseCreatedMessage(RealtimeServerMessageType type) /// public string? OutputVoice { get; set; } - /// - /// Gets or sets the conversation ID associated with the response. - /// - /// - /// Identifies which conversation within the session this response belongs to. - /// A session may have a default conversation to which items are automatically added, - /// or responses may be generated out-of-band (not associated with any conversation). - /// - public string? ConversationId { get; set; } - /// /// Gets or sets the unique response ID. /// diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 1363ae85142..153771f1ab7 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -1035,7 +1035,6 @@ private static RealtimeServerResponseCreatedMessage MapResponseCreatedOrDone( } msg.ResponseId = response.Id; - msg.ConversationId = response.ConversationId; msg.Status = response.Status?.ToString(); if (response.AudioOptions?.OutputAudioOptions is { } audioOut) diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs index 6745febdd01..8ef5262f720 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs @@ -963,11 +963,6 @@ private void TraceStreamingResponse( _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.Id, response.ResponseId); } - if (!string.IsNullOrWhiteSpace(response.ConversationId)) - { - _ = activity.AddTag(OpenTelemetryConsts.GenAI.Conversation.Id, response.ConversationId); - } - if (!string.IsNullOrWhiteSpace(response.Status)) { _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.FinishReasons, $"[\"{response.Status}\"]"); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs index 0d70dbb4a77..5ce8dcbce74 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs @@ -174,7 +174,6 @@ public void ResponseCreatedMessage_DefaultProperties() Assert.Null(message.OutputAudioOptions); Assert.Null(message.OutputVoice); - Assert.Null(message.ConversationId); Assert.Null(message.ResponseId); Assert.Null(message.MaxOutputTokens); Assert.Null(message.AdditionalProperties); @@ -202,7 +201,6 @@ public void ResponseCreatedMessage_Properties_Roundtrip() { OutputAudioOptions = audioFormat, OutputVoice = "alloy", - ConversationId = "conv_1", ResponseId = "resp_1", MaxOutputTokens = 1000, AdditionalProperties = metadata, @@ -215,7 +213,6 @@ public void ResponseCreatedMessage_Properties_Roundtrip() Assert.Same(audioFormat, message.OutputAudioOptions); Assert.Equal("alloy", message.OutputVoice); - Assert.Equal("conv_1", message.ConversationId); Assert.Equal("resp_1", message.ResponseId); Assert.Equal(1000, message.MaxOutputTokens); Assert.Same(metadata, message.AdditionalProperties); diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs index 211c84dabd7..c054cd5c80a 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs @@ -60,7 +60,6 @@ static async IAsyncEnumerable CallbackAsync([EnumeratorCa yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone) { ResponseId = "resp_12345", - ConversationId = "conv_67890", Status = "completed", Usage = new UsageDetails { @@ -132,7 +131,6 @@ static async IAsyncEnumerable CallbackAsync([EnumeratorCa // Response attributes Assert.Equal("resp_12345", activity.GetTagItem("gen_ai.response.id")); - Assert.Equal("conv_67890", activity.GetTagItem("gen_ai.conversation.id")); Assert.Equal("""["completed"]""", activity.GetTagItem("gen_ai.response.finish_reasons")); Assert.Equal(15, activity.GetTagItem("gen_ai.usage.input_tokens")); Assert.Equal(25, activity.GetTagItem("gen_ai.usage.output_tokens")); From f372c42ddadadd922f1c9a643a15f71a7720555c Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Wed, 4 Mar 2026 17:25:04 -0800 Subject: [PATCH 56/92] Rename ResponseCreateMessage and ConversationItemCreateMessage to CreateResponseMessage and CreateConversationItemMessage --- ...timeClientCreateConversationItemMessage.cs} | 6 +++--- ... => RealtimeClientCreateResponseMessage.cs} | 6 +++--- .../OpenAIRealtimeSession.cs | 8 ++++---- .../FunctionInvokingRealtimeSession.cs | 16 ++++++++-------- .../Realtime/OpenTelemetryRealtimeSession.cs | 4 ++-- .../Realtime/RealtimeClientMessageTests.cs | 14 +++++++------- .../FunctionInvokingRealtimeSessionTests.cs | 14 +++++++------- .../OpenTelemetryRealtimeSessionTests.cs | 18 +++++++++--------- 8 files changed, 43 insertions(+), 43 deletions(-) rename src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/{RealtimeClientConversationItemCreateMessage.cs => RealtimeClientCreateConversationItemMessage.cs} (88%) rename src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/{RealtimeClientResponseCreateMessage.cs => RealtimeClientCreateResponseMessage.cs} (96%) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateConversationItemMessage.cs similarity index 88% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateConversationItemMessage.cs index dc10766e347..9e07c10a1a1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientConversationItemCreateMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateConversationItemMessage.cs @@ -11,14 +11,14 @@ namespace Microsoft.Extensions.AI; /// Represents a real-time message for creating a conversation item. /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class RealtimeClientConversationItemCreateMessage : RealtimeClientMessage +public class RealtimeClientCreateConversationItemMessage : RealtimeClientMessage { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The conversation item to create. /// The optional ID of the previous conversation item to insert the new one after. - public RealtimeClientConversationItemCreateMessage(RealtimeContentItem item, string? previousId = null) + public RealtimeClientCreateConversationItemMessage(RealtimeContentItem item, string? previousId = null) { PreviousId = previousId; Item = Throw.IfNull(item); diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateResponseMessage.cs similarity index 96% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateResponseMessage.cs index c74e568f2d3..12f367601b0 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientResponseCreateMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateResponseMessage.cs @@ -17,12 +17,12 @@ namespace Microsoft.Extensions.AI; /// for this response only. /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class RealtimeClientResponseCreateMessage : RealtimeClientMessage +public class RealtimeClientCreateResponseMessage : RealtimeClientMessage { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - public RealtimeClientResponseCreateMessage() + public RealtimeClientCreateResponseMessage() { } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 153771f1ab7..1c88cf16030 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -127,11 +127,11 @@ public async Task SendAsync(RealtimeClientMessage message, CancellationToken can { switch (message) { - case RealtimeClientResponseCreateMessage responseCreate: + case RealtimeClientCreateResponseMessage responseCreate: await SendResponseCreateAsync(responseCreate, cancellationToken).ConfigureAwait(false); break; - case RealtimeClientConversationItemCreateMessage itemCreate: + case RealtimeClientCreateConversationItemMessage itemCreate: await SendConversationItemCreateAsync(itemCreate, cancellationToken).ConfigureAwait(false); break; @@ -209,7 +209,7 @@ public ValueTask DisposeAsync() #region Send Helpers (MEAI → SDK) - private async Task SendResponseCreateAsync(RealtimeClientResponseCreateMessage responseCreate, CancellationToken cancellationToken) + private async Task SendResponseCreateAsync(RealtimeClientCreateResponseMessage responseCreate, CancellationToken cancellationToken) { var responseOptions = new Sdk.RealtimeResponseOptions(); @@ -308,7 +308,7 @@ private async Task SendResponseCreateAsync(RealtimeClientResponseCreateMessage r } } - private async Task SendConversationItemCreateAsync(RealtimeClientConversationItemCreateMessage itemCreate, CancellationToken cancellationToken) + private async Task SendConversationItemCreateAsync(RealtimeClientCreateConversationItemMessage itemCreate, CancellationToken cancellationToken) { if (itemCreate.Item is null) { diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs index b9de08e8ea1..14db9f80e24 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs @@ -22,14 +22,14 @@ namespace Microsoft.Extensions.AI; /// -/// A delegating realtime session that invokes functions defined on . +/// A delegating realtime session that invokes functions defined on . /// Include this in a realtime session pipeline to resolve function calls automatically. /// /// /// /// When this session receives a in a realtime server message from its inner /// , it responds by invoking the corresponding defined -/// in (or in ), producing a +/// in (or in ), producing a /// that it sends back to the inner session. This loop is repeated until there are no more function calls to make, or until /// another stop condition is met, such as hitting . /// @@ -41,7 +41,7 @@ namespace Microsoft.Extensions.AI; /// /// /// A instance is thread-safe for concurrent use so long as the -/// instances employed as part of the supplied are also safe. +/// instances employed as part of the supplied are also safe. /// The property can be used to control whether multiple function invocation /// requests as part of the same request are invocable concurrently, but even with that set to /// (the default), multiple concurrent requests to this same instance and using the same tools could result in those @@ -212,8 +212,8 @@ public int MaximumConsecutiveErrorsPerRequest /// Gets or sets a collection of additional tools the session is able to invoke. /// /// These will not impact the requests sent by the , which will pass through the - /// unmodified. However, if the inner session requests the invocation of a tool - /// that was not in , this collection will also be consulted + /// unmodified. However, if the inner session requests the invocation of a tool + /// that was not in , this collection will also be consulted /// to look for a corresponding tool to invoke. This is useful when the service might have been preconfigured to be aware /// of certain tools that aren't also sent on each individual request. /// @@ -238,7 +238,7 @@ public int MaximumConsecutiveErrorsPerRequest /// /// /// s that the is aware of (for example, because they're in - /// or ) but that aren't s aren't considered + /// or ) but that aren't s aren't considered /// unknown, just not invocable. Any requests to a non-invocable tool will also result in the function calling loop terminating, /// regardless of . /// @@ -458,14 +458,14 @@ private List CreateFunctionResultMessages(List { new TextContent("Hello") }; var item = new RealtimeContentItem(contents, "item_1", ChatRole.User); - var message = new RealtimeClientConversationItemCreateMessage(item, "prev_1"); + var message = new RealtimeClientCreateConversationItemMessage(item, "prev_1"); Assert.Same(item, message.Item); Assert.Equal("prev_1", message.PreviousId); @@ -50,7 +50,7 @@ public void ConversationItemCreateMessage_Constructor_SetsProperties() public void ConversationItemCreateMessage_Constructor_PreviousIdDefaults() { var item = new RealtimeContentItem([new TextContent("Hello")]); - var message = new RealtimeClientConversationItemCreateMessage(item); + var message = new RealtimeClientCreateConversationItemMessage(item); Assert.Same(item, message.Item); Assert.Null(message.PreviousId); @@ -60,7 +60,7 @@ public void ConversationItemCreateMessage_Constructor_PreviousIdDefaults() public void ConversationItemCreateMessage_Properties_Roundtrip() { var item = new RealtimeContentItem([new TextContent("Hello")]); - var message = new RealtimeClientConversationItemCreateMessage(item); + var message = new RealtimeClientCreateConversationItemMessage(item); var newItem = new RealtimeContentItem([new TextContent("World")]); message.Item = newItem; @@ -74,7 +74,7 @@ public void ConversationItemCreateMessage_Properties_Roundtrip() public void ConversationItemCreateMessage_InheritsClientMessage() { var item = new RealtimeContentItem([new TextContent("Hello")]); - var message = new RealtimeClientConversationItemCreateMessage(item) + var message = new RealtimeClientCreateConversationItemMessage(item) { MessageId = "evt_create_1", }; @@ -129,7 +129,7 @@ public void InputAudioBufferCommitMessage_Constructor() [Fact] public void ResponseCreateMessage_DefaultProperties() { - var message = new RealtimeClientResponseCreateMessage(); + var message = new RealtimeClientCreateResponseMessage(); Assert.Null(message.Items); Assert.Null(message.OutputAudioOptions); @@ -146,7 +146,7 @@ public void ResponseCreateMessage_DefaultProperties() [Fact] public void ResponseCreateMessage_Properties_Roundtrip() { - var message = new RealtimeClientResponseCreateMessage(); + var message = new RealtimeClientCreateResponseMessage(); var items = new List { @@ -183,7 +183,7 @@ public void ResponseCreateMessage_Properties_Roundtrip() [Fact] public void ResponseCreateMessage_InheritsClientMessage() { - var message = new RealtimeClientResponseCreateMessage + var message = new RealtimeClientCreateResponseMessage { MessageId = "evt_resp_1", RawRepresentation = "raw", diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs index d751c8067d4..03fac7c232a 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs @@ -125,14 +125,14 @@ public async Task GetStreamingResponseAsync_FunctionCall_InvokesAndInjectsResult Assert.Equal(2, injectedMessages.Count); // First injected: conversation.item.create with function result - var resultMsg = Assert.IsType(injectedMessages[0]); + var resultMsg = Assert.IsType(injectedMessages[0]); Assert.NotNull(resultMsg.Item); var functionResult = Assert.IsType(resultMsg.Item.Contents[0]); Assert.Equal("call_001", functionResult.CallId); Assert.Contains("Sunny in Seattle", functionResult.Result?.ToString()); // Second injected: response.create (no hardcoded modalities) - var responseCreate = Assert.IsType(injectedMessages[1]); + var responseCreate = Assert.IsType(injectedMessages[1]); Assert.Null(responseCreate.OutputModalities); } @@ -169,7 +169,7 @@ public async Task GetStreamingResponseAsync_FunctionCall_FromAdditionalTools() } Assert.Equal(2, injectedMessages.Count); - var resultMsg = Assert.IsType(injectedMessages[0]); + var resultMsg = Assert.IsType(injectedMessages[0]); var functionResult = Assert.IsType(resultMsg.Item.Contents[0]); Assert.Contains("Rainy in London", functionResult.Result?.ToString()); } @@ -277,7 +277,7 @@ public async Task GetStreamingResponseAsync_UnknownFunction_SendsErrorByDefault( // Should inject error result + response.create Assert.Equal(2, injectedMessages.Count); - var resultMsg = Assert.IsType(injectedMessages[0]); + var resultMsg = Assert.IsType(injectedMessages[0]); var functionResult = Assert.IsType(resultMsg.Item.Contents[0]); Assert.Contains("not found", functionResult.Result?.ToString(), StringComparison.OrdinalIgnoreCase); } @@ -316,7 +316,7 @@ public async Task GetStreamingResponseAsync_FunctionError_IncludesDetailedErrors } Assert.Equal(2, injectedMessages.Count); - var resultMsg = Assert.IsType(injectedMessages[0]); + var resultMsg = Assert.IsType(injectedMessages[0]); var functionResult = Assert.IsType(resultMsg.Item.Contents[0]); Assert.Contains("Something broke", functionResult.Result?.ToString()); } @@ -354,7 +354,7 @@ public async Task GetStreamingResponseAsync_FunctionError_HidesDetailsWhenNotEna // consume } - var resultMsg = Assert.IsType(injectedMessages[0]); + var resultMsg = Assert.IsType(injectedMessages[0]); var functionResult = Assert.IsType(resultMsg.Item.Contents[0]); Assert.DoesNotContain("Secret error info", functionResult.Result?.ToString()); Assert.Contains("failed", functionResult.Result?.ToString(), StringComparison.OrdinalIgnoreCase); @@ -439,7 +439,7 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_False_SendsE // Error result + response.create should be injected (default behavior) Assert.Equal(2, injectedMessages.Count); - var resultMsg = Assert.IsType(injectedMessages[0]); + var resultMsg = Assert.IsType(injectedMessages[0]); var functionResult = Assert.IsType(resultMsg.Item.Contents[0]); Assert.Contains("not found", functionResult.Result?.ToString(), StringComparison.OrdinalIgnoreCase); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs index c054cd5c80a..c3200236d8e 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs @@ -843,7 +843,7 @@ private static async IAsyncEnumerable GetClientMessagesAs await Task.Yield(); yield return new RealtimeClientInputAudioBufferAppendMessage(new DataContent(new byte[] { 1, 2, 3 }, "audio/pcm")); yield return new RealtimeClientInputAudioBufferCommitMessage(); - yield return new RealtimeClientResponseCreateMessage(); + yield return new RealtimeClientCreateResponseMessage(); } #pragma warning disable IDE0060 // Remove unused parameter @@ -852,8 +852,8 @@ private static async IAsyncEnumerable GetClientMessagesWi { await Task.Yield(); var contentItem = new RealtimeContentItem([new FunctionResultContent("call_1", "result_value")], role: ChatRole.Tool); - yield return new RealtimeClientConversationItemCreateMessage(contentItem); - yield return new RealtimeClientResponseCreateMessage(); + yield return new RealtimeClientCreateConversationItemMessage(contentItem); + yield return new RealtimeClientCreateResponseMessage(); } private static async IAsyncEnumerable CallbackWithToolCallAsync([EnumeratorCancellation] CancellationToken cancellationToken) @@ -1254,7 +1254,7 @@ private static async IAsyncEnumerable GetClientMessagesWi #pragma warning restore IDE0060 { await Task.Yield(); - yield return new RealtimeClientResponseCreateMessage { Instructions = "Be very helpful" }; + yield return new RealtimeClientCreateResponseMessage { Instructions = "Be very helpful" }; } #pragma warning disable IDE0060 // Remove unused parameter @@ -1263,7 +1263,7 @@ private static async IAsyncEnumerable GetClientMessagesWi { await Task.Yield(); var item = new RealtimeContentItem([new TextContent("Hello from client")], role: ChatRole.User); - yield return new RealtimeClientResponseCreateMessage { Items = [item] }; + yield return new RealtimeClientCreateResponseMessage { Items = [item] }; } #pragma warning disable IDE0060 // Remove unused parameter @@ -1272,8 +1272,8 @@ private static async IAsyncEnumerable GetClientMessagesWi { await Task.Yield(); var item = new RealtimeContentItem([new TextContent("User text message")], role: ChatRole.User); - yield return new RealtimeClientConversationItemCreateMessage(item); - yield return new RealtimeClientResponseCreateMessage(); + yield return new RealtimeClientCreateConversationItemMessage(item); + yield return new RealtimeClientCreateResponseMessage(); } #pragma warning disable IDE0060 // Remove unused parameter @@ -1283,8 +1283,8 @@ private static async IAsyncEnumerable GetClientMessagesWi await Task.Yield(); var imageData = new DataContent(new byte[] { 0x89, 0x50, 0x4E, 0x47 }, "image/png"); var item = new RealtimeContentItem([imageData], role: ChatRole.User); - yield return new RealtimeClientConversationItemCreateMessage(item); - yield return new RealtimeClientResponseCreateMessage(); + yield return new RealtimeClientCreateConversationItemMessage(item); + yield return new RealtimeClientCreateResponseMessage(); } private static async IAsyncEnumerable CallbackWithTextOutputAsync([EnumeratorCancellation] CancellationToken cancellationToken) From 0c8fe651fa4d81da254ff836fea3a801341860a0 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Thu, 5 Mar 2026 09:36:25 -0800 Subject: [PATCH 57/92] Rename RealtimeContentItem to RealtimeConversationItem --- ...timeClientCreateConversationItemMessage.cs | 4 +-- .../RealtimeClientCreateResponseMessage.cs | 2 +- ...entItem.cs => RealtimeConversationItem.cs} | 6 ++-- .../RealtimeServerResponseCreatedMessage.cs | 2 +- ...RealtimeServerResponseOutputItemMessage.cs | 2 +- .../OpenAIRealtimeSession.cs | 34 +++++++++---------- .../FunctionInvokingRealtimeSession.cs | 4 +-- .../Realtime/OpenTelemetryRealtimeSession.cs | 2 +- .../Realtime/RealtimeClientMessageTests.cs | 14 ++++---- ...ts.cs => RealtimeConversationItemTests.cs} | 10 +++--- .../Realtime/RealtimeServerMessageTests.cs | 6 ++-- .../FunctionInvokingRealtimeSessionTests.cs | 4 +-- .../OpenTelemetryRealtimeSessionTests.cs | 10 +++--- 13 files changed, 50 insertions(+), 50 deletions(-) rename src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/{RealtimeContentItem.cs => RealtimeConversationItem.cs} (93%) rename test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/{RealtimeContentItemTests.cs => RealtimeConversationItemTests.cs} (86%) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateConversationItemMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateConversationItemMessage.cs index 9e07c10a1a1..44edff595f7 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateConversationItemMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateConversationItemMessage.cs @@ -18,7 +18,7 @@ public class RealtimeClientCreateConversationItemMessage : RealtimeClientMessage /// /// The conversation item to create. /// The optional ID of the previous conversation item to insert the new one after. - public RealtimeClientCreateConversationItemMessage(RealtimeContentItem item, string? previousId = null) + public RealtimeClientCreateConversationItemMessage(RealtimeConversationItem item, string? previousId = null) { PreviousId = previousId; Item = Throw.IfNull(item); @@ -33,5 +33,5 @@ public RealtimeClientCreateConversationItemMessage(RealtimeContentItem item, str /// /// Gets or sets the conversation item to create. /// - public RealtimeContentItem Item { get; set; } + public RealtimeConversationItem Item { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateResponseMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateResponseMessage.cs index 12f367601b0..bdfe74ac316 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateResponseMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateResponseMessage.cs @@ -29,7 +29,7 @@ public RealtimeClientCreateResponseMessage() /// /// Gets or sets the list of the conversation items to create a response for. /// - public IList? Items { get; set; } + public IList? Items { get; set; } /// /// Gets or sets the output audio options for the response. diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeConversationItem.cs similarity index 93% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeConversationItem.cs index f6d6d7e9c39..7373a5d6773 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeContentItem.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeConversationItem.cs @@ -15,15 +15,15 @@ namespace Microsoft.Extensions.AI; /// or sent as part of a real-time response creation process. /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class RealtimeContentItem +public class RealtimeConversationItem { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The contents of the conversation item. /// The ID of the conversation item. /// The role of the conversation item. - public RealtimeContentItem(IList contents, string? id = null, ChatRole? role = null) + public RealtimeConversationItem(IList contents, string? id = null, ChatRole? role = null) { Id = id; Role = role; diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs index 8c946186db1..b46336f4df6 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs @@ -72,7 +72,7 @@ public RealtimeServerResponseCreatedMessage(RealtimeServerMessageType type) /// /// Gets or sets the list of the conversation items included in the response. /// - public IList? Items { get; set; } + public IList? Items { get; set; } /// /// Gets or sets the output modalities for the response. like "text", "audio". diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs index 632d2261005..0fec5899340 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs @@ -47,5 +47,5 @@ public RealtimeServerResponseOutputItemMessage(RealtimeServerMessageType type) /// /// Gets or sets the conversation item included in the response. /// - public RealtimeContentItem? Item { get; set; } + public RealtimeConversationItem? Item { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 1c88cf16030..d1c807e23d9 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -684,7 +684,7 @@ private static Sdk.RealtimeMcpToolCallApprovalPolicy ToSdkCustomApprovalPolicy(H return custom; } - private static Sdk.RealtimeItem? ToRealtimeItem(RealtimeContentItem? contentItem) + private static Sdk.RealtimeItem? ToRealtimeItem(RealtimeConversationItem? contentItem) { if (contentItem?.Contents is null or { Count: 0 }) { @@ -1082,10 +1082,10 @@ private static RealtimeServerResponseCreatedMessage MapResponseCreatedOrDone( if (response.OutputItems is { Count: > 0 } outputItems) { - var items = new List(); + var items = new List(); foreach (var item in outputItems) { - if (MapRealtimeItem(item) is RealtimeContentItem contentItem) + if (MapRealtimeItem(item) is RealtimeConversationItem contentItem) { items.Add(contentItem); } @@ -1184,7 +1184,7 @@ private static RealtimeServerResponseOutputItemMessage MapMcpCallEvent( return new RealtimeServerResponseOutputItemMessage(type) { MessageId = eventId, - Item = itemId is not null ? new RealtimeContentItem([], itemId) : null, + Item = itemId is not null ? new RealtimeConversationItem([], itemId) : null, OutputIndex = outputIndex, RawRepresentation = update, }; @@ -1196,16 +1196,16 @@ private static RealtimeServerResponseOutputItemMessage MapMcpListToolsEvent( return new RealtimeServerResponseOutputItemMessage(type) { MessageId = eventId, - Item = itemId is not null ? new RealtimeContentItem([], itemId) : null, + Item = itemId is not null ? new RealtimeConversationItem([], itemId) : null, RawRepresentation = update, }; } - private static RealtimeContentItem? MapRealtimeItem(Sdk.RealtimeItem item) => item switch + private static RealtimeConversationItem? MapRealtimeItem(Sdk.RealtimeItem item) => item switch { Sdk.RealtimeMessageItem messageItem => MapMessageItem(messageItem), Sdk.RealtimeFunctionCallItem funcCallItem => MapFunctionCallItem(funcCallItem), - Sdk.RealtimeFunctionCallOutputItem funcOutputItem => new RealtimeContentItem( + Sdk.RealtimeFunctionCallOutputItem funcOutputItem => new RealtimeConversationItem( [new FunctionResultContent(funcOutputItem.CallId ?? string.Empty, funcOutputItem.FunctionOutput)], funcOutputItem.Id), Sdk.RealtimeMcpToolCallItem mcpItem => MapMcpToolCallItem(mcpItem), @@ -1214,17 +1214,17 @@ [new FunctionResultContent(funcOutputItem.CallId ?? string.Empty, funcOutputItem _ => null, }; - private static RealtimeContentItem MapFunctionCallItem(Sdk.RealtimeFunctionCallItem funcCallItem) + private static RealtimeConversationItem MapFunctionCallItem(Sdk.RealtimeFunctionCallItem funcCallItem) { var arguments = funcCallItem.FunctionArguments is not null && !funcCallItem.FunctionArguments.IsEmpty ? JsonSerializer.Deserialize>(funcCallItem.FunctionArguments) : null; - return new RealtimeContentItem( + return new RealtimeConversationItem( [new FunctionCallContent(funcCallItem.CallId ?? string.Empty, funcCallItem.FunctionName, arguments)], funcCallItem.Id); } - private static RealtimeContentItem MapMessageItem(Sdk.RealtimeMessageItem messageItem) + private static RealtimeConversationItem MapMessageItem(Sdk.RealtimeMessageItem messageItem) { var contents = new List(); if (messageItem.Content is not null) @@ -1270,10 +1270,10 @@ private static RealtimeContentItem MapMessageItem(Sdk.RealtimeMessageItem messag : messageItem.Role == Sdk.RealtimeMessageRole.System ? ChatRole.System : null; - return new RealtimeContentItem(contents, messageItem.Id, role); + return new RealtimeConversationItem(contents, messageItem.Id, role); } - private static RealtimeContentItem MapMcpToolCallItem(Sdk.RealtimeMcpToolCallItem mcpItem) + private static RealtimeConversationItem MapMcpToolCallItem(Sdk.RealtimeMcpToolCallItem mcpItem) { string callId = mcpItem.Id ?? string.Empty; @@ -1309,10 +1309,10 @@ private static RealtimeContentItem MapMcpToolCallItem(Sdk.RealtimeMcpToolCallIte }); } - return new RealtimeContentItem(contents, mcpItem.Id); + return new RealtimeConversationItem(contents, mcpItem.Id); } - private static RealtimeContentItem MapMcpApprovalRequestItem(Sdk.RealtimeMcpToolCallApprovalRequestItem approvalItem) + private static RealtimeConversationItem MapMcpApprovalRequestItem(Sdk.RealtimeMcpToolCallApprovalRequestItem approvalItem) { string approvalId = approvalItem.Id ?? string.Empty; @@ -1332,12 +1332,12 @@ private static RealtimeContentItem MapMcpApprovalRequestItem(Sdk.RealtimeMcpTool RawRepresentation = approvalItem, }; - return new RealtimeContentItem( + return new RealtimeConversationItem( [new McpServerToolApprovalRequestContent(approvalId, toolCall) { RawRepresentation = approvalItem }], approvalItem.Id); } - private static RealtimeContentItem MapMcpToolDefinitionListItem(Sdk.RealtimeMcpToolDefinitionListItem toolListItem) + private static RealtimeConversationItem MapMcpToolDefinitionListItem(Sdk.RealtimeMcpToolDefinitionListItem toolListItem) { var contents = new List(); foreach (var toolDef in toolListItem.ToolDefinitions) @@ -1351,7 +1351,7 @@ private static RealtimeContentItem MapMcpToolDefinitionListItem(Sdk.RealtimeMcpT } } - return new RealtimeContentItem(contents, toolListItem.Id); + return new RealtimeConversationItem(contents, toolListItem.Id); } private static UsageDetails? MapUsageDetails(Sdk.RealtimeResponseUsage? usage) diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs index 14db9f80e24..3a58376d89a 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs @@ -454,8 +454,8 @@ private List CreateFunctionResultMessages(List message } /// Extracts content from an AIContent list and converts to OTel format. - private RealtimeOtelMessage? ExtractOtelMessage(RealtimeContentItem? item) + private RealtimeOtelMessage? ExtractOtelMessage(RealtimeConversationItem? item) { if (item?.Contents is null or { Count: 0 }) { diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs index 6b0243409ae..f6204bd72a4 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs @@ -38,7 +38,7 @@ public void RealtimeClientMessage_Properties_Roundtrip() public void ConversationItemCreateMessage_Constructor_SetsProperties() { var contents = new List { new TextContent("Hello") }; - var item = new RealtimeContentItem(contents, "item_1", ChatRole.User); + var item = new RealtimeConversationItem(contents, "item_1", ChatRole.User); var message = new RealtimeClientCreateConversationItemMessage(item, "prev_1"); @@ -49,7 +49,7 @@ public void ConversationItemCreateMessage_Constructor_SetsProperties() [Fact] public void ConversationItemCreateMessage_Constructor_PreviousIdDefaults() { - var item = new RealtimeContentItem([new TextContent("Hello")]); + var item = new RealtimeConversationItem([new TextContent("Hello")]); var message = new RealtimeClientCreateConversationItemMessage(item); Assert.Same(item, message.Item); @@ -59,10 +59,10 @@ public void ConversationItemCreateMessage_Constructor_PreviousIdDefaults() [Fact] public void ConversationItemCreateMessage_Properties_Roundtrip() { - var item = new RealtimeContentItem([new TextContent("Hello")]); + var item = new RealtimeConversationItem([new TextContent("Hello")]); var message = new RealtimeClientCreateConversationItemMessage(item); - var newItem = new RealtimeContentItem([new TextContent("World")]); + var newItem = new RealtimeConversationItem([new TextContent("World")]); message.Item = newItem; message.PreviousId = "prev_2"; @@ -73,7 +73,7 @@ public void ConversationItemCreateMessage_Properties_Roundtrip() [Fact] public void ConversationItemCreateMessage_InheritsClientMessage() { - var item = new RealtimeContentItem([new TextContent("Hello")]); + var item = new RealtimeConversationItem([new TextContent("Hello")]); var message = new RealtimeClientCreateConversationItemMessage(item) { MessageId = "evt_create_1", @@ -148,9 +148,9 @@ public void ResponseCreateMessage_Properties_Roundtrip() { var message = new RealtimeClientCreateResponseMessage(); - var items = new List + var items = new List { - new RealtimeContentItem([new TextContent("Hello")], "item_1", ChatRole.User), + new RealtimeConversationItem([new TextContent("Hello")], "item_1", ChatRole.User), }; var audioFormat = new RealtimeAudioFormat("audio/pcm", 16000); var modalities = new List { "text", "audio" }; diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeContentItemTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeConversationItemTests.cs similarity index 86% rename from test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeContentItemTests.cs rename to test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeConversationItemTests.cs index c2e50936894..0beae48edf0 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeContentItemTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeConversationItemTests.cs @@ -8,13 +8,13 @@ namespace Microsoft.Extensions.AI; -public class RealtimeContentItemTests +public class RealtimeConversationItemTests { [Fact] public void Constructor_WithContentsOnly_PropsDefaulted() { IList contents = [new TextContent("Hello")]; - var item = new RealtimeContentItem(contents); + var item = new RealtimeConversationItem(contents); Assert.Same(contents, item.Contents); Assert.Null(item.Id); @@ -26,7 +26,7 @@ public void Constructor_WithContentsOnly_PropsDefaulted() public void Constructor_WithAllArgs_PropsRoundtrip() { IList contents = [new TextContent("Hello"), new TextContent("World")]; - var item = new RealtimeContentItem(contents, "item_123", ChatRole.User); + var item = new RealtimeConversationItem(contents, "item_123", ChatRole.User); Assert.Same(contents, item.Contents); Assert.Equal("item_123", item.Id); @@ -37,7 +37,7 @@ public void Constructor_WithAllArgs_PropsRoundtrip() public void Properties_Roundtrip() { IList contents = [new TextContent("Initial")]; - var item = new RealtimeContentItem(contents); + var item = new RealtimeConversationItem(contents); IList newContents = [new TextContent("Updated")]; item.Id = "new_id"; @@ -56,7 +56,7 @@ public void Constructor_WithFunctionContent_NoIdOrRole() { var functionCall = new FunctionCallContent("call_1", "myFunc"); IList contents = [functionCall]; - var item = new RealtimeContentItem(contents); + var item = new RealtimeConversationItem(contents); Assert.Null(item.Id); Assert.Null(item.Role); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs index 5ce8dcbce74..2f9b2a489f7 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs @@ -189,9 +189,9 @@ public void ResponseCreatedMessage_Properties_Roundtrip() { var audioFormat = new RealtimeAudioFormat("audio/pcm", 24000); var metadata = new AdditionalPropertiesDictionary { ["key"] = "value" }; - var items = new List + var items = new List { - new RealtimeContentItem([new TextContent("response")], "item_1"), + new RealtimeConversationItem([new TextContent("response")], "item_1"), }; var modalities = new List { "text" }; var error = new ErrorContent("response error"); @@ -245,7 +245,7 @@ public void ResponseOutputItemMessage_DefaultProperties() [Fact] public void ResponseOutputItemMessage_Properties_Roundtrip() { - var item = new RealtimeContentItem([new TextContent("output")], "item_out_1", ChatRole.Assistant); + var item = new RealtimeConversationItem([new TextContent("output")], "item_out_1", ChatRole.Assistant); var message = new RealtimeServerResponseOutputItemMessage(RealtimeServerMessageType.ResponseOutputItemDone) { diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs index 03fac7c232a..a3924ae507a 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs @@ -482,7 +482,7 @@ public async Task GetStreamingResponseAsync_ConcurrentInvocation_InvokesInParall var msg2 = CreateFunctionCallOutputItemMessage("call_b", "slow_func", null); // Combine both into a single ResponseOutputItem with multiple function calls - var combinedItem = new RealtimeContentItem( + var combinedItem = new RealtimeConversationItem( [ new FunctionCallContent("call_a", "slow_func"), new FunctionCallContent("call_b", "slow_func"), @@ -637,7 +637,7 @@ private static RealtimeServerResponseOutputItemMessage CreateFunctionCallOutputI string callId, string functionName, IDictionary? arguments) { var functionCallContent = new FunctionCallContent(callId, functionName, arguments); - var item = new RealtimeContentItem([functionCallContent], $"item_{callId}"); + var item = new RealtimeConversationItem([functionCallContent], $"item_{callId}"); return new RealtimeServerResponseOutputItemMessage(RealtimeServerMessageType.ResponseOutputItemDone) { diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs index c3200236d8e..81080b24dfe 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs @@ -851,7 +851,7 @@ private static async IAsyncEnumerable GetClientMessagesWi #pragma warning restore IDE0060 { await Task.Yield(); - var contentItem = new RealtimeContentItem([new FunctionResultContent("call_1", "result_value")], role: ChatRole.Tool); + var contentItem = new RealtimeConversationItem([new FunctionResultContent("call_1", "result_value")], role: ChatRole.Tool); yield return new RealtimeClientCreateConversationItemMessage(contentItem); yield return new RealtimeClientCreateResponseMessage(); } @@ -862,7 +862,7 @@ private static async IAsyncEnumerable CallbackWithToolCal _ = cancellationToken; // Yield a function call item from the server using RealtimeServerResponseOutputItemMessage - var contentItem = new RealtimeContentItem( + var contentItem = new RealtimeConversationItem( [new FunctionCallContent("call_123", "search", new Dictionary { ["query"] = "test" })], role: ChatRole.Assistant); yield return new RealtimeServerResponseOutputItemMessage(RealtimeServerMessageType.ResponseOutputItemDone) @@ -1262,7 +1262,7 @@ private static async IAsyncEnumerable GetClientMessagesWi #pragma warning restore IDE0060 { await Task.Yield(); - var item = new RealtimeContentItem([new TextContent("Hello from client")], role: ChatRole.User); + var item = new RealtimeConversationItem([new TextContent("Hello from client")], role: ChatRole.User); yield return new RealtimeClientCreateResponseMessage { Items = [item] }; } @@ -1271,7 +1271,7 @@ private static async IAsyncEnumerable GetClientMessagesWi #pragma warning restore IDE0060 { await Task.Yield(); - var item = new RealtimeContentItem([new TextContent("User text message")], role: ChatRole.User); + var item = new RealtimeConversationItem([new TextContent("User text message")], role: ChatRole.User); yield return new RealtimeClientCreateConversationItemMessage(item); yield return new RealtimeClientCreateResponseMessage(); } @@ -1282,7 +1282,7 @@ private static async IAsyncEnumerable GetClientMessagesWi { await Task.Yield(); var imageData = new DataContent(new byte[] { 0x89, 0x50, 0x4E, 0x47 }, "image/png"); - var item = new RealtimeContentItem([imageData], role: ChatRole.User); + var item = new RealtimeConversationItem([imageData], role: ChatRole.User); yield return new RealtimeClientCreateConversationItemMessage(item); yield return new RealtimeClientCreateResponseMessage(); } From df8d2c0a2b9b1fa7755feb77b0fafb1f2e025777 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Thu, 5 Mar 2026 12:28:26 -0800 Subject: [PATCH 58/92] Remove VoiceSpeed from RealtimeSessionOptions and use RawRepresentationFactory seed pattern --- .../Realtime/RealtimeSessionOptions.cs | 8 --- .../OpenAIRealtimeSession.cs | 29 +++-------- .../OpenTelemetryConsts.cs | 6 --- .../Realtime/OpenTelemetryRealtimeSession.cs | 7 --- .../Realtime/RealtimeSessionOptionsTests.cs | 3 -- .../OpenTelemetryRealtimeSessionTests.cs | 50 ------------------- 6 files changed, 8 insertions(+), 95 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs index bff02b1dc0b..9474eaa6294 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs @@ -51,14 +51,6 @@ public class RealtimeSessionOptions /// public RealtimeAudioFormat? OutputAudioFormat { get; init; } - /// - /// Gets the output voice speed for the session. - /// - /// - /// The default value is 1.0, which represents normal speed. - /// - public double VoiceSpeed { get; init; } = 1.0; - /// /// Gets the output voice for the session. /// diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index d1c807e23d9..3609a413a05 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -90,11 +90,7 @@ public async Task UpdateAsync(RealtimeSessionOptions options, CancellationToken // Allow callers to provide a pre-configured SDK-specific options instance. object? rawOptions = options.RawRepresentationFactory?.Invoke(this); - if (rawOptions is Sdk.RealtimeConversationSessionOptions rawConvOptions) - { - await _sessionClient.ConfigureConversationSessionAsync(rawConvOptions, cancellationToken).ConfigureAwait(false); - } - else if (rawOptions is Sdk.RealtimeTranscriptionSessionOptions rawTransOptions) + if (rawOptions is Sdk.RealtimeTranscriptionSessionOptions rawTransOptions) { await _sessionClient.ConfigureTranscriptionSessionAsync(rawTransOptions, cancellationToken).ConfigureAwait(false); } @@ -105,7 +101,7 @@ public async Task UpdateAsync(RealtimeSessionOptions options, CancellationToken } else { - var convOpts = BuildConversationSessionOptions(options); + var convOpts = BuildConversationSessionOptions(options, rawOptions as Sdk.RealtimeConversationSessionOptions); await _sessionClient.ConfigureConversationSessionAsync(convOpts, cancellationToken).ConfigureAwait(false); } } @@ -383,14 +379,14 @@ private async Task SendRawCommandAsync(RealtimeClientMessage message, Cancellati } } - private static Sdk.RealtimeConversationSessionOptions BuildConversationSessionOptions(RealtimeSessionOptions options) + private static Sdk.RealtimeConversationSessionOptions BuildConversationSessionOptions(RealtimeSessionOptions options, Sdk.RealtimeConversationSessionOptions? seedOptions = null) { - var convOptions = new Sdk.RealtimeConversationSessionOptions(); + var convOptions = seedOptions ?? new Sdk.RealtimeConversationSessionOptions(); - // Audio configuration. - var audioOptions = new Sdk.RealtimeConversationSessionAudioOptions(); - var inputAudioOptions = new Sdk.RealtimeConversationSessionInputAudioOptions(); - var outputAudioOptions = new Sdk.RealtimeConversationSessionOutputAudioOptions(); + // Audio configuration. Reuse any existing options from the seed to preserve provider-specific settings. + var audioOptions = convOptions.AudioOptions ?? new Sdk.RealtimeConversationSessionAudioOptions(); + var inputAudioOptions = audioOptions.InputAudioOptions ?? new Sdk.RealtimeConversationSessionInputAudioOptions(); + var outputAudioOptions = audioOptions.OutputAudioOptions ?? new Sdk.RealtimeConversationSessionOutputAudioOptions(); if (options.InputAudioFormat is not null) { @@ -457,8 +453,6 @@ private static Sdk.RealtimeConversationSessionOptions BuildConversationSessionOp outputAudioOptions.AudioFormat = ToSdkAudioFormat(options.OutputAudioFormat); } - outputAudioOptions.Speed = (float)options.VoiceSpeed; - if (options.Voice is not null) { outputAudioOptions.Voice = new Sdk.RealtimeVoice(options.Voice); @@ -913,7 +907,6 @@ private RealtimeSessionOptions MapConversationSessionToOptions(Sdk.RealtimeConve TranscriptionOptions? transcription = null; VoiceActivityDetection? vad = null; RealtimeAudioFormat? outputAudioFormat = null; - double voiceSpeed = 1.0; string? voice = null; if (session.AudioOptions is { } audioOptions) @@ -975,11 +968,6 @@ private RealtimeSessionOptions MapConversationSessionToOptions(Sdk.RealtimeConve { outputAudioFormat = MapSdkAudioFormat(outputOpts.AudioFormat); - if (outputOpts.Speed.HasValue) - { - voiceSpeed = outputOpts.Speed.Value; - } - if (outputOpts.Voice.HasValue) { voice = outputOpts.Voice.Value.ToString(); @@ -1011,7 +999,6 @@ private RealtimeSessionOptions MapConversationSessionToOptions(Sdk.RealtimeConve TranscriptionOptions = transcription, VoiceActivityDetection = vad, OutputAudioFormat = outputAudioFormat, - VoiceSpeed = voiceSpeed, Voice = voice, // Preserve client-side properties that the server cannot round-trip. diff --git a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs index a20aae7560a..77c1320ac3e 100644 --- a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs +++ b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs @@ -191,12 +191,6 @@ public static class Realtime /// public const string Voice = "gen_ai.realtime.voice"; - /// - /// The speed multiplier for voice output in a realtime session (e.g., 1.0 for normal speed). - /// Custom attribute: "gen_ai.realtime.voice_speed". - /// - public const string VoiceSpeed = "gen_ai.realtime.voice_speed"; - /// /// The output modalities configured for a realtime session (e.g., "Text", "Audio"). /// Custom attribute: "gen_ai.realtime.output_modalities". diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs index e1fde8a8885..962a1beb6df 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs @@ -778,13 +778,6 @@ private static string SerializeMessages(IEnumerable message _ = activity.AddTag(OpenTelemetryConsts.GenAI.Realtime.Voice, options.Voice); } -#pragma warning disable S1244 // Floating point numbers should not be tested for equality - if (options.VoiceSpeed != 1.0) -#pragma warning restore S1244 - { - _ = activity.AddTag(OpenTelemetryConsts.GenAI.Realtime.VoiceSpeed, options.VoiceSpeed); - } - if (options.OutputModalities is { Count: > 0 } modalities) { _ = activity.AddTag(OpenTelemetryConsts.GenAI.Realtime.OutputModalities, $"[{string.Join(", ", modalities.Select(m => $"\"{m}\""))}]"); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs index 2e50aba62c4..ae2f005e454 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs @@ -22,7 +22,6 @@ public void Constructor_Parameterless_PropsDefaulted() Assert.Null(options.TranscriptionOptions); Assert.Null(options.VoiceActivityDetection); Assert.Null(options.OutputAudioFormat); - Assert.Equal(1.0, options.VoiceSpeed); Assert.Null(options.Voice); Assert.Null(options.Instructions); Assert.Null(options.MaxOutputTokens); @@ -50,7 +49,6 @@ public void Properties_Roundtrip() NoiseReductionOptions = NoiseReductionOptions.NearField, TranscriptionOptions = transcriptionOptions, VoiceActivityDetection = vad, - VoiceSpeed = 1.5, Voice = "alloy", Instructions = "Be helpful", MaxOutputTokens = 500, @@ -66,7 +64,6 @@ public void Properties_Roundtrip() Assert.Equal(NoiseReductionOptions.NearField, options.NoiseReductionOptions); Assert.Same(transcriptionOptions, options.TranscriptionOptions); Assert.Same(vad, options.VoiceActivityDetection); - Assert.Equal(1.5, options.VoiceSpeed); Assert.Equal("alloy", options.Voice); Assert.Equal("Be helpful", options.Instructions); Assert.Equal(500, options.MaxOutputTokens); diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs index 81080b24dfe..7bb99675bf4 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs @@ -34,7 +34,6 @@ public async Task ExpectedInformationLogged_GetStreamingResponseAsync(bool enabl { Model = "test-model", Voice = "alloy", - VoiceSpeed = 1.2, MaxOutputTokens = 500, OutputModalities = ["text", "audio"], Instructions = "Be helpful and friendly.", @@ -126,7 +125,6 @@ static async IAsyncEnumerable CallbackAsync([EnumeratorCa // Realtime-specific attributes Assert.Equal("Realtime", activity.GetTagItem("gen_ai.realtime.session_kind")); Assert.Equal("alloy", activity.GetTagItem("gen_ai.realtime.voice")); - Assert.Equal(1.2, activity.GetTagItem("gen_ai.realtime.voice_speed")); Assert.Equal("""["text", "audio"]""", activity.GetTagItem("gen_ai.realtime.output_modalities")); // Response attributes @@ -351,54 +349,6 @@ static async IAsyncEnumerable ErrorResponseCallbackAsync( Assert.Equal("Something went wrong", activity.StatusDescription); } - [Fact] - public async Task DefaultVoiceSpeed_NotLogged() - { - var sourceName = Guid.NewGuid().ToString(); - var activities = new List(); - using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() - .AddSource(sourceName) - .AddInMemoryExporter(activities) - .Build(); - - await using var innerSession = new TestRealtimeSession - { - Options = new RealtimeSessionOptions - { - Model = "test-model", - VoiceSpeed = 1.0, // Default value should not be logged - }, - GetStreamingResponseAsyncCallback = (cancellationToken) => EmptyCallbackAsync(cancellationToken), - }; - - static async IAsyncEnumerable EmptyCallbackAsync([EnumeratorCancellation] CancellationToken cancellationToken) - { - await Task.Yield(); - _ = cancellationToken; - - yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); - } - - await using var session = innerSession - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName) - .Build(); - - await foreach (var msg in GetClientMessagesAsync()) - { - await session.SendAsync(msg); - } - - await foreach (var response in session.GetStreamingResponseAsync()) - { - // Consume - } - - var activity = Assert.Single(activities); - var tags = activity.Tags.ToDictionary(kvp => kvp.Key, kvp => kvp.Value); - Assert.False(tags.ContainsKey("gen_ai.realtime.voice_speed")); - } - [Fact] public async Task NoListeners_NoActivityCreated() { From 0e9c3b8bc24c836fc321755e3ba9d54a848c3d9b Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Thu, 5 Mar 2026 13:05:27 -0800 Subject: [PATCH 59/92] Remove UpdateAsync from IRealtimeClientSession, use message-based approach Replace UpdateAsync method on IRealtimeClientSession with a new RealtimeClientSessionUpdateMessage type sent via SendAsync. This avoids requiring all providers to implement session updates (e.g. Gemini does not support mid-session updates). - Add RealtimeClientSessionUpdateMessage with Options property - Remove UpdateAsync from IRealtimeClientSession and DelegatingRealtimeSession - Move update logic to OpenAIRealtimeSession.SendAsync handler - Remove UpdateAsync overrides from Logging and OpenTelemetry sessions - Update all tests to use SendAsync with the new message type --- .../Realtime/DelegatingRealtimeSession.cs | 4 - .../Realtime/IRealtimeClientSession.cs | 6 -- .../RealtimeClientSessionUpdateMessage.cs | 40 +++++++++ .../Realtime/RealtimeSessionOptions.cs | 2 +- .../OpenAIRealtimeClient.cs | 2 +- .../OpenAIRealtimeSession.cs | 9 +- .../Realtime/LoggingRealtimeSession.cs | 36 -------- .../Realtime/OpenTelemetryRealtimeSession.cs | 54 ------------ .../TestRealtimeSession.cs | 9 -- .../OpenAIRealtimeSessionTests.cs | 5 +- .../DelegatingRealtimeSessionTests.cs | 17 ++-- .../Realtime/LoggingRealtimeSessionTests.cs | 37 ++++----- .../OpenTelemetryRealtimeSessionTests.cs | 83 +------------------ 13 files changed, 76 insertions(+), 228 deletions(-) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientSessionUpdateMessage.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs index aeeb46a04a8..b7eb75262dd 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs @@ -57,10 +57,6 @@ protected virtual async ValueTask DisposeAsyncCore() public virtual Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => InnerSession.SendAsync(message, cancellationToken); - /// - public virtual Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancellationToken = default) => - InnerSession.UpdateAsync(options, cancellationToken); - /// public virtual IAsyncEnumerable GetStreamingResponseAsync( CancellationToken cancellationToken = default) => diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClientSession.cs index 1632a0c91f5..0240262c513 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClientSession.cs @@ -15,12 +15,6 @@ namespace Microsoft.Extensions.AI; [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] public interface IRealtimeClientSession : IAsyncDisposable { - /// Updates the session with new options. - /// The new session options. - /// A token to cancel the operation. - /// A task that represents the asynchronous update operation. - Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancellationToken = default); - /// /// Gets the current session options. /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientSessionUpdateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientSessionUpdateMessage.cs new file mode 100644 index 00000000000..6d0f75def20 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientSessionUpdateMessage.cs @@ -0,0 +1,40 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a client message that requests updating the session configuration. +/// +/// +/// +/// Sending this message requests that the provider update the active session with new options. +/// Not all providers support mid-session updates. Providers that do not support this message +/// may ignore it or throw a . +/// +/// +/// When a provider processes this message, it should update its +/// property to reflect the new configuration. +/// +/// +[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] +public class RealtimeClientSessionUpdateMessage : RealtimeClientMessage +{ + /// + /// Initializes a new instance of the class. + /// + /// The session options to apply. + public RealtimeClientSessionUpdateMessage(RealtimeSessionOptions options) + { + Options = Throw.IfNull(options); + } + + /// + /// Gets or sets the session options to apply. + /// + public RealtimeSessionOptions Options { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs index 9474eaa6294..65c77dc1ef3 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs @@ -87,7 +87,7 @@ public class RealtimeSessionOptions /// /// /// The underlying implementation might have its own representation of options. - /// When is invoked with a , + /// When a is sent with a , /// that implementation might convert the provided options into its own representation in order to use it while /// performing the operation. For situations where a consumer knows which concrete /// is being used and how it represents options, a new instance of that implementation-specific options type can be diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs index 5a3182547f9..ebab59dd201 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs @@ -63,7 +63,7 @@ public async Task CreateSessionAsync(RealtimeSessionOpti { if (options is not null) { - await session.UpdateAsync(options, cancellationToken).ConfigureAwait(false); + await session.SendAsync(new RealtimeClientSessionUpdateMessage(options), cancellationToken).ConfigureAwait(false); } return session; diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs index 3609a413a05..5fa78b59138 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs @@ -80,11 +80,8 @@ public async Task ConnectAsync(CancellationToken cancellationToken = default) _model, cancellationToken: cancellationToken).ConfigureAwait(false); } - /// - public async Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancellationToken = default) + private async Task UpdateSessionAsync(RealtimeSessionOptions options, CancellationToken cancellationToken) { - _ = Throw.IfNull(options); - if (_sessionClient is not null) { // Allow callers to provide a pre-configured SDK-specific options instance. @@ -123,6 +120,10 @@ public async Task SendAsync(RealtimeClientMessage message, CancellationToken can { switch (message) { + case RealtimeClientSessionUpdateMessage sessionUpdate: + await UpdateSessionAsync(sessionUpdate.Options, cancellationToken).ConfigureAwait(false); + break; + case RealtimeClientCreateResponseMessage responseCreate: await SendResponseCreateAsync(responseCreate, cancellationToken).ConfigureAwait(false); break; diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs index 12e6183a464..7f379606062 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs @@ -53,42 +53,6 @@ public JsonSerializerOptions JsonSerializerOptions set => _jsonSerializerOptions = Throw.IfNull(value); } - /// - public override async Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancellationToken = default) - { - if (_logger.IsEnabled(LogLevel.Debug)) - { - if (_logger.IsEnabled(LogLevel.Trace)) - { - LogInvokedSensitive(nameof(UpdateAsync), AsJson(options)); - } - else - { - LogInvoked(nameof(UpdateAsync)); - } - } - - try - { - await base.UpdateAsync(options, cancellationToken).ConfigureAwait(false); - - if (_logger.IsEnabled(LogLevel.Debug)) - { - LogCompleted(nameof(UpdateAsync)); - } - } - catch (OperationCanceledException) - { - LogInvocationCanceled(nameof(UpdateAsync)); - throw; - } - catch (Exception ex) - { - LogInvocationFailed(nameof(UpdateAsync), ex); - throw; - } - } - /// public override async Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) { diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs index 962a1beb6df..f44698a1381 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs @@ -169,32 +169,6 @@ protected override async ValueTask DisposeAsyncCore() serviceType == typeof(ActivitySource) ? _activitySource : base.GetService(serviceType, serviceKey); - /// - public override async Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(options); - _jsonSerializerOptions.MakeReadOnly(); - - using Activity? activity = CreateAndConfigureActivity(options); - Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; - string? requestModelId = options.Model ?? _defaultModelId; - - Exception? error = null; - try - { - await base.UpdateAsync(options, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - error = ex; - throw; - } - finally - { - TraceUpdateResponse(activity, requestModelId, error, stopwatch); - } - } - /// public override async Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) { @@ -833,34 +807,6 @@ private static string SerializeMessages(IEnumerable message return activity; } - /// Adds update operation response information to the activity. - private void TraceUpdateResponse( - Activity? activity, - string? requestModelId, - Exception? error, - Stopwatch? stopwatch) - { - if (_operationDurationHistogram.Enabled && stopwatch is not null) - { - TagList tags = default; - AddMetricTags(ref tags, requestModelId, responseModelId: null); - - if (error is not null) - { - tags.Add(OpenTelemetryConsts.Error.Type, error.GetType().FullName); - } - - _operationDurationHistogram.Record(stopwatch.Elapsed.TotalSeconds, tags); - } - - if (error is not null) - { - _ = activity? - .AddTag(OpenTelemetryConsts.Error.Type, error.GetType().FullName) - .SetStatus(ActivityStatusCode.Error, error.Message); - } - } - /// Adds streaming response information to the activity. private void TraceStreamingResponse( Activity? activity, diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs index 3a2ae3acec8..82fccc46230 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs @@ -11,9 +11,6 @@ namespace Microsoft.Extensions.AI; /// A test implementation that uses callbacks for verification. public sealed class TestRealtimeSession : IRealtimeClientSession { - /// Gets or sets the callback to invoke when is called. - public Func? UpdateAsyncCallback { get; set; } - /// Gets or sets the callback to invoke when is called. public Func? SendAsyncCallback { get; set; } @@ -26,12 +23,6 @@ public sealed class TestRealtimeSession : IRealtimeClientSession /// public RealtimeSessionOptions? Options { get; set; } - /// - public Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancellationToken = default) - { - return UpdateAsyncCallback?.Invoke(options, cancellationToken) ?? Task.CompletedTask; - } - /// public Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) { diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs index 3730a03f2f2..a6188b8fdfc 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs @@ -49,10 +49,9 @@ public async Task Options_InitiallyNull() } [Fact] - public async Task UpdateAsync_NullOptions_Throws() + public void SessionUpdateMessage_NullOptions_Throws() { - await using var session = new OpenAIRealtimeSession("key", "model"); - await Assert.ThrowsAsync("options", () => session.UpdateAsync(null!)); + Assert.Throws("options", () => new RealtimeClientSessionUpdateMessage(null!)); } [Fact] diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs index 19c6298e592..c2c950c0951 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs @@ -31,22 +31,23 @@ public async Task Options_DelegatesToInner() } [Fact] - public async Task UpdateAsync_DelegatesToInner() + public async Task SendAsync_SessionUpdateMessage_DelegatesToInner() { var called = false; var sentOptions = new RealtimeSessionOptions { Instructions = "Be helpful" }; await using var inner = new TestRealtimeSession { - UpdateAsyncCallback = (options, _) => + SendAsyncCallback = (msg, _) => { - Assert.Same(sentOptions, options); + var updateMsg = Assert.IsType(msg); + Assert.Same(sentOptions, updateMsg.Options); called = true; return Task.CompletedTask; }, }; await using var delegating = new NoOpDelegatingRealtimeSession(inner); - await delegating.UpdateAsync(sentOptions); + await delegating.SendAsync(new RealtimeClientSessionUpdateMessage(sentOptions)); Assert.True(called); } @@ -143,7 +144,7 @@ public async Task DisposeAsync_DisposesInner() } [Fact] - public async Task UpdateAsync_FlowsCancellationToken() + public async Task SendAsync_SessionUpdateMessage_FlowsCancellationToken() { CancellationToken capturedToken = default; using var cts = new CancellationTokenSource(); @@ -151,7 +152,7 @@ public async Task UpdateAsync_FlowsCancellationToken() await using var inner = new TestRealtimeSession { - UpdateAsyncCallback = (options, ct) => + SendAsyncCallback = (msg, ct) => { capturedToken = ct; return Task.CompletedTask; @@ -159,7 +160,7 @@ public async Task UpdateAsync_FlowsCancellationToken() }; await using var delegating = new NoOpDelegatingRealtimeSession(inner); - await delegating.UpdateAsync(sentOptions, cts.Token); + await delegating.SendAsync(new RealtimeClientSessionUpdateMessage(sentOptions), cts.Token); Assert.Equal(cts.Token, capturedToken); } @@ -214,8 +215,6 @@ public DisposableTestRealtimeSession(Action onDispose) public RealtimeSessionOptions? Options => null; - public Task UpdateAsync(RealtimeSessionOptions options, CancellationToken cancellationToken = default) => Task.CompletedTask; - public Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => Task.CompletedTask; public IAsyncEnumerable GetStreamingResponseAsync( diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs index 8f6b662cb12..a46543064c4 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs @@ -47,7 +47,7 @@ public async Task UseLogging_AvoidsInjectingNopSession() [InlineData(LogLevel.Trace)] [InlineData(LogLevel.Debug)] [InlineData(LogLevel.Information)] - public async Task UpdateAsync_LogsInvocationAndCompletion(LogLevel level) + public async Task SendAsync_SessionUpdateMessage_LogsInvocationAndCompletion(LogLevel level) { var collector = new FakeLogCollector(); @@ -55,30 +55,27 @@ public async Task UpdateAsync_LogsInvocationAndCompletion(LogLevel level) c.AddLogging(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(level)); var services = c.BuildServiceProvider(); - await using var innerSession = new TestRealtimeSession - { - UpdateAsyncCallback = (options, cancellationToken) => Task.CompletedTask, - }; + await using var innerSession = new TestRealtimeSession(); await using var session = innerSession .AsBuilder() .UseLogging() .Build(services); - await session.UpdateAsync(new RealtimeSessionOptions { Model = "test-model", Instructions = "Be helpful" }); + await session.SendAsync(new RealtimeClientSessionUpdateMessage(new RealtimeSessionOptions { Model = "test-model", Instructions = "Be helpful" })); var logs = collector.GetSnapshot(); if (level is LogLevel.Trace) { Assert.Collection(logs, - entry => Assert.Contains("UpdateAsync invoked:", entry.Message), - entry => Assert.Contains("UpdateAsync completed.", entry.Message)); + entry => Assert.Contains("SendAsync invoked:", entry.Message), + entry => Assert.Contains("SendAsync completed.", entry.Message)); } else if (level is LogLevel.Debug) { Assert.Collection(logs, - entry => Assert.Contains("UpdateAsync invoked.", entry.Message), - entry => Assert.Contains("UpdateAsync completed.", entry.Message)); + entry => Assert.Contains("SendAsync invoked.", entry.Message), + entry => Assert.Contains("SendAsync completed.", entry.Message)); } else { @@ -184,7 +181,7 @@ static async IAsyncEnumerable GetMessagesAsync() } [Fact] - public async Task UpdateAsync_LogsCancellation() + public async Task SendAsync_SessionUpdateMessage_LogsCancellation() { var collector = new FakeLogCollector(); using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); @@ -193,7 +190,7 @@ public async Task UpdateAsync_LogsCancellation() await using var innerSession = new TestRealtimeSession { - UpdateAsyncCallback = (options, cancellationToken) => + SendAsyncCallback = (msg, cancellationToken) => { throw new OperationCanceledException(cancellationToken); }, @@ -205,23 +202,23 @@ public async Task UpdateAsync_LogsCancellation() .Build(); cts.Cancel(); - await Assert.ThrowsAsync(() => session.UpdateAsync(new RealtimeSessionOptions(), cts.Token)); + await Assert.ThrowsAsync(() => session.SendAsync(new RealtimeClientSessionUpdateMessage(new RealtimeSessionOptions()), cts.Token)); var logs = collector.GetSnapshot(); Assert.Collection(logs, - entry => Assert.Contains("UpdateAsync invoked.", entry.Message), - entry => Assert.Contains("UpdateAsync canceled.", entry.Message)); + entry => Assert.Contains("SendAsync invoked.", entry.Message), + entry => Assert.Contains("SendAsync canceled.", entry.Message)); } [Fact] - public async Task UpdateAsync_LogsErrors() + public async Task SendAsync_SessionUpdateMessage_LogsErrors() { var collector = new FakeLogCollector(); using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); await using var innerSession = new TestRealtimeSession { - UpdateAsyncCallback = (options, cancellationToken) => + SendAsyncCallback = (msg, cancellationToken) => { throw new InvalidOperationException("Test error"); }, @@ -232,12 +229,12 @@ public async Task UpdateAsync_LogsErrors() .UseLogging(loggerFactory) .Build(); - await Assert.ThrowsAsync(() => session.UpdateAsync(new RealtimeSessionOptions())); + await Assert.ThrowsAsync(() => session.SendAsync(new RealtimeClientSessionUpdateMessage(new RealtimeSessionOptions()))); var logs = collector.GetSnapshot(); Assert.Collection(logs, - entry => Assert.Contains("UpdateAsync invoked.", entry.Message), - entry => Assert.True(entry.Message.Contains("UpdateAsync failed.") && entry.Level == LogLevel.Error)); + entry => Assert.Contains("SendAsync invoked.", entry.Message), + entry => Assert.True(entry.Message.Contains("SendAsync failed.") && entry.Level == LogLevel.Error)); } [Fact] diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs index 7bb99675bf4..4f12dfba8f3 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs @@ -180,82 +180,6 @@ static async IAsyncEnumerable CallbackAsync([EnumeratorCa } } - [Fact] - public async Task UpdateAsync_TracesOperation() - { - var sourceName = Guid.NewGuid().ToString(); - var activities = new List(); - using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() - .AddSource(sourceName) - .AddInMemoryExporter(activities) - .Build(); - - await using var innerSession = new TestRealtimeSession - { - UpdateAsyncCallback = (options, cancellationToken) => Task.CompletedTask, - GetServiceCallback = (serviceType, serviceKey) => - serviceType == typeof(ChatClientMetadata) ? new ChatClientMetadata("testprovider", new Uri("http://localhost:8080"), "gpt-4-realtime") : - null, - }; - - await using var session = innerSession - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName, configure: instance => - { - instance.EnableSensitiveData = true; - }) - .Build(); - - var options = new RealtimeSessionOptions - { - Model = "my-model", - Voice = "echo", - MaxOutputTokens = 100, - Instructions = "Be brief.", - }; - - await session.UpdateAsync(options); - - var activity = Assert.Single(activities); - - Assert.NotNull(activity.Id); - Assert.Equal("realtime my-model", activity.DisplayName); - Assert.Equal("chat", activity.GetTagItem("gen_ai.operation.name")); - Assert.Equal("my-model", activity.GetTagItem("gen_ai.request.model")); - Assert.Equal("testprovider", activity.GetTagItem("gen_ai.provider.name")); - Assert.Equal("echo", activity.GetTagItem("gen_ai.realtime.voice")); - Assert.Equal(100, activity.GetTagItem("gen_ai.request.max_tokens")); - Assert.True(activity.Duration.TotalMilliseconds > 0); - } - - [Fact] - public async Task UpdateAsync_TracesError() - { - var sourceName = Guid.NewGuid().ToString(); - var activities = new List(); - using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() - .AddSource(sourceName) - .AddInMemoryExporter(activities) - .Build(); - - await using var innerSession = new TestRealtimeSession - { - UpdateAsyncCallback = (options, cancellationToken) => throw new InvalidOperationException("Test error"), - }; - - await using var session = innerSession - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName) - .Build(); - - await Assert.ThrowsAsync(() => session.UpdateAsync(new RealtimeSessionOptions { Model = "test" })); - - var activity = Assert.Single(activities); - Assert.Equal("System.InvalidOperationException", activity.GetTagItem("error.type")); - Assert.Equal(ActivityStatusCode.Error, activity.Status); - Assert.Equal("Test error", activity.StatusDescription); - } - [Fact] public async Task GetStreamingResponseAsync_TracesError() { @@ -400,12 +324,9 @@ public async Task InvalidArgs_Throws() } [Fact] - public async Task UpdateAsync_InvalidArgs_Throws() + public void SessionUpdateMessage_NullOptions_Throws() { - await using var innerSession = new TestRealtimeSession(); - await using var session = new OpenTelemetryRealtimeSession(innerSession); - - await Assert.ThrowsAsync("options", () => session.UpdateAsync(null!)); + Assert.Throws("options", () => new RealtimeClientSessionUpdateMessage(null!)); } [Fact] From 5ffae0c43b30c1986b65a105702789e802d48188 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Thu, 5 Mar 2026 13:52:17 -0800 Subject: [PATCH 60/92] Rename realtime session classes to include Client in name Align implementation class names with IRealtimeClientSession interface naming. All session-related classes now include 'Client': - DelegatingRealtimeSession -> DelegatingRealtimeClientSession - OpenAIRealtimeSession -> OpenAIRealtimeClientSession - LoggingRealtimeSession -> LoggingRealtimeClientSession - OpenTelemetryRealtimeSession -> OpenTelemetryRealtimeClientSession - FunctionInvokingRealtimeSession -> FunctionInvokingRealtimeClientSession - RealtimeSessionBuilder -> RealtimeClientSessionBuilder - RealtimeSessionExtensions -> RealtimeClientSessionExtensions - TestRealtimeSession -> TestRealtimeClientSession - All related builder extension and test classes updated --- ....cs => DelegatingRealtimeClientSession.cs} | 6 +- .../Realtime/RealtimeServerMessageType.cs | 4 +- .../RealtimeServerResponseCreatedMessage.cs | 2 +- ...RealtimeServerResponseOutputItemMessage.cs | 2 +- .../OpenAIRealtimeClient.cs | 2 +- ...sion.cs => OpenAIRealtimeClientSession.cs} | 10 +-- .../Common/FunctionInvocationHelpers.cs | 2 +- .../Common/FunctionInvocationLogger.cs | 2 +- .../Common/FunctionInvocationProcessor.cs | 2 +- ...onymousDelegatingRealtimeClientSession.cs} | 6 +- ... FunctionInvokingRealtimeClientSession.cs} | 28 +++--- ...RealtimeClientSessionBuilderExtensions.cs} | 18 ++-- ...ion.cs => LoggingRealtimeClientSession.cs} | 6 +- ...RealtimeClientSessionBuilderExtensions.cs} | 18 ++-- ... => OpenTelemetryRealtimeClientSession.cs} | 6 +- ...RealtimeClientSessionBuilderExtensions.cs} | 16 ++-- ...der.cs => RealtimeClientSessionBuilder.cs} | 26 +++--- ...BuilderRealtimeClientSessionExtensions.cs} | 14 +-- ....cs => RealtimeClientSessionExtensions.cs} | 2 +- ...ession.cs => TestRealtimeClientSession.cs} | 2 +- ...cs => OpenAIRealtimeClientSessionTests.cs} | 20 ++--- .../Microsoft.Extensions.AI.Tests.csproj | 2 +- ...> DelegatingRealtimeClientSessionTests.cs} | 68 +++++++-------- ...tionInvokingRealtimeClientSessionTests.cs} | 86 +++++++++---------- ...s => LoggingRealtimeClientSessionTests.cs} | 56 ++++++------ ...penTelemetryRealtimeClientSessionTests.cs} | 74 ++++++++-------- ...s => RealtimeClientSessionBuilderTests.cs} | 56 ++++++------ ...> RealtimeClientSessionExtensionsTests.cs} | 14 +-- 28 files changed, 275 insertions(+), 275 deletions(-) rename src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/{DelegatingRealtimeSession.cs => DelegatingRealtimeClientSession.cs} (94%) rename src/Libraries/Microsoft.Extensions.AI.OpenAI/{OpenAIRealtimeSession.cs => OpenAIRealtimeClientSession.cs} (99%) rename src/Libraries/Microsoft.Extensions.AI/Realtime/{AnonymousDelegatingRealtimeSession.cs => AnonymousDelegatingRealtimeClientSession.cs} (90%) rename src/Libraries/Microsoft.Extensions.AI/Realtime/{FunctionInvokingRealtimeSession.cs => FunctionInvokingRealtimeClientSession.cs} (95%) rename src/Libraries/Microsoft.Extensions.AI/Realtime/{FunctionInvokingRealtimeSessionBuilderExtensions.cs => FunctionInvokingRealtimeClientSessionBuilderExtensions.cs} (64%) rename src/Libraries/Microsoft.Extensions.AI/Realtime/{LoggingRealtimeSession.cs => LoggingRealtimeClientSession.cs} (97%) rename src/Libraries/Microsoft.Extensions.AI/Realtime/{LoggingRealtimeSessionBuilderExtensions.cs => LoggingRealtimeClientSessionBuilderExtensions.cs} (76%) rename src/Libraries/Microsoft.Extensions.AI/Realtime/{OpenTelemetryRealtimeSession.cs => OpenTelemetryRealtimeClientSession.cs} (99%) rename src/Libraries/Microsoft.Extensions.AI/Realtime/{OpenTelemetryRealtimeSessionBuilderExtensions.cs => OpenTelemetryRealtimeClientSessionBuilderExtensions.cs} (84%) rename src/Libraries/Microsoft.Extensions.AI/Realtime/{RealtimeSessionBuilder.cs => RealtimeClientSessionBuilder.cs} (82%) rename src/Libraries/Microsoft.Extensions.AI/Realtime/{RealtimeSessionBuilderRealtimeSessionExtensions.cs => RealtimeClientSessionBuilderRealtimeClientSessionExtensions.cs} (60%) rename src/Libraries/Microsoft.Extensions.AI/Realtime/{RealtimeSessionExtensions.cs => RealtimeClientSessionExtensions.cs} (96%) rename test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/{TestRealtimeSession.cs => TestRealtimeClientSession.cs} (96%) rename test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/{OpenAIRealtimeSessionTests.cs => OpenAIRealtimeClientSessionTests.cs} (77%) rename test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/{DelegatingRealtimeSessionTests.cs => DelegatingRealtimeClientSessionTests.cs} (71%) rename test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/{FunctionInvokingRealtimeSessionTests.cs => FunctionInvokingRealtimeClientSessionTests.cs} (88%) rename test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/{LoggingRealtimeSessionTests.cs => LoggingRealtimeClientSessionTests.cs} (88%) rename test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/{OpenTelemetryRealtimeSessionTests.cs => OpenTelemetryRealtimeClientSessionTests.cs} (94%) rename test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/{RealtimeSessionBuilderTests.cs => RealtimeClientSessionBuilderTests.cs} (72%) rename test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/{RealtimeSessionExtensionsTests.cs => RealtimeClientSessionExtensionsTests.cs} (71%) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeClientSession.cs similarity index 94% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeClientSession.cs index b7eb75262dd..8018915dc7c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeClientSession.cs @@ -19,14 +19,14 @@ namespace Microsoft.Extensions.AI; /// The default implementation simply passes each call to the inner session instance. /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class DelegatingRealtimeSession : IRealtimeClientSession +public class DelegatingRealtimeClientSession : IRealtimeClientSession { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The wrapped session instance. /// is . - protected DelegatingRealtimeSession(IRealtimeClientSession innerSession) + protected DelegatingRealtimeClientSession(IRealtimeClientSession innerSession) { InnerSession = Throw.IfNull(innerSession); } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs index 455f38631e7..d7ceb0d52ca 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs @@ -23,8 +23,8 @@ namespace Microsoft.Extensions.AI; /// /// /// Provider implementations that want to support the built-in middleware pipeline -/// ( and -/// ) must emit the following +/// ( and +/// ) must emit the following /// message types at appropriate points during response generation: /// /// — when the model begins generating a new response. diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs index b46336f4df6..a6934d45f32 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs @@ -17,7 +17,7 @@ namespace Microsoft.Extensions.AI; /// /// Provider implementations should emit this message with /// when the model begins generating a new response, and with -/// when the response is complete. The built-in middleware depends +/// when the response is complete. The built-in middleware depends /// on these messages for tracing response lifecycle. /// /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs index 0fec5899340..38e9143b075 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs @@ -16,7 +16,7 @@ namespace Microsoft.Extensions.AI; /// /// Provider implementations should emit this message with /// when an output item (such as a function call or text message) has completed. The built-in -/// middleware depends on this message to detect +/// middleware depends on this message to detect /// and invoke tool calls. /// /// diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs index ebab59dd201..05dfc20b469 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs @@ -58,7 +58,7 @@ public async Task CreateSessionAsync(RealtimeSessionOpti ? await _realtimeClient.StartTranscriptionSessionAsync(cancellationToken: cancellationToken).ConfigureAwait(false) : await _realtimeClient.StartConversationSessionAsync(_model, cancellationToken: cancellationToken).ConfigureAwait(false); - var session = new OpenAIRealtimeSession(sessionClient, _model); + var session = new OpenAIRealtimeClientSession(sessionClient, _model); try { if (options is not null) diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs similarity index 99% rename from src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs rename to src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs index 5fa78b59138..f0655b99743 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs @@ -25,7 +25,7 @@ namespace Microsoft.Extensions.AI; /// Represents an for the OpenAI Realtime API over WebSocket. [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public sealed class OpenAIRealtimeSession : IRealtimeClientSession +public sealed class OpenAIRealtimeClientSession : IRealtimeClientSession { /// The model to use for the session. private readonly string _model; @@ -45,20 +45,20 @@ public sealed class OpenAIRealtimeSession : IRealtimeClientSession /// public RealtimeSessionOptions? Options { get; private set; } - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// The API key used for authentication. /// The model to use for the session. - public OpenAIRealtimeSession(string apiKey, string model) + public OpenAIRealtimeClientSession(string apiKey, string model) { _ownedRealtimeClient = new Sdk.RealtimeClient(Throw.IfNull(apiKey)); _model = Throw.IfNull(model); _metadata = new("openai", defaultModelId: _model); } - /// Initializes a new instance of the class from an already-connected session client. + /// Initializes a new instance of the class from an already-connected session client. /// The connected SDK session client. /// The model name for metadata. - internal OpenAIRealtimeSession(Sdk.RealtimeSessionClient sessionClient, string model) + internal OpenAIRealtimeClientSession(Sdk.RealtimeSessionClient sessionClient, string model) { _sessionClient = Throw.IfNull(sessionClient); _model = model ?? string.Empty; diff --git a/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationHelpers.cs b/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationHelpers.cs index 45fa6cf78c0..de2b3cc745c 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationHelpers.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationHelpers.cs @@ -8,7 +8,7 @@ namespace Microsoft.Extensions.AI; /// -/// Internal helper methods shared between and . +/// Internal helper methods shared between and . /// internal static class FunctionInvocationHelpers { diff --git a/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationLogger.cs b/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationLogger.cs index df5a38c4fbd..1e811432195 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationLogger.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationLogger.cs @@ -7,7 +7,7 @@ namespace Microsoft.Extensions.AI; /// -/// Internal logger for function invocation operations shared between and . +/// Internal logger for function invocation operations shared between and . /// internal static partial class FunctionInvocationLogger { diff --git a/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationProcessor.cs b/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationProcessor.cs index 0d8bd959e45..19b472a5e5d 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationProcessor.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationProcessor.cs @@ -16,7 +16,7 @@ namespace Microsoft.Extensions.AI; /// /// A composition-based helper class for processing function invocations. -/// Used by both and . +/// Used by both and . /// internal sealed class FunctionInvocationProcessor { diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeClientSession.cs similarity index 90% rename from src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeSession.cs rename to src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeClientSession.cs index 6635a72a1d3..8ee05cd05aa 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeClientSession.cs @@ -11,13 +11,13 @@ namespace Microsoft.Extensions.AI; /// Represents a delegating realtime session that wraps an inner session with implementations provided by delegates. [Experimental("MEAI001")] -internal sealed class AnonymousDelegatingRealtimeSession : DelegatingRealtimeSession +internal sealed class AnonymousDelegatingRealtimeClientSession : DelegatingRealtimeClientSession { /// The delegate to use as the implementation of . private readonly Func> _getStreamingResponseFunc; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The inner session. /// @@ -25,7 +25,7 @@ internal sealed class AnonymousDelegatingRealtimeSession : DelegatingRealtimeSes /// /// is . /// is . - public AnonymousDelegatingRealtimeSession( + public AnonymousDelegatingRealtimeClientSession( IRealtimeClientSession innerSession, Func> getStreamingResponseFunc) : base(innerSession) diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs similarity index 95% rename from src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs rename to src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs index 3a58376d89a..1133777b4fa 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs @@ -35,12 +35,12 @@ namespace Microsoft.Extensions.AI; /// /// /// If a requested function is an but not an , the -/// will not attempt to invoke it, and instead allow that +/// will not attempt to invoke it, and instead allow that /// to pass back out to the caller. It is then that caller's responsibility to create the appropriate /// for that call and send it back as part of a subsequent request. /// /// -/// A instance is thread-safe for concurrent use so long as the +/// A instance is thread-safe for concurrent use so long as the /// instances employed as part of the supplied are also safe. /// The property can be used to control whether multiple function invocation /// requests as part of the same request are invocable concurrently, but even with that set to @@ -49,12 +49,12 @@ namespace Microsoft.Extensions.AI; /// /// [Experimental("MEAI001")] -public class FunctionInvokingRealtimeSession : DelegatingRealtimeSession +public class FunctionInvokingRealtimeClientSession : DelegatingRealtimeClientSession { /// The for the current function invocation. private static readonly AsyncLocal _currentContext = new(); - /// Gets the specified when constructing the , if any. + /// Gets the specified when constructing the , if any. protected IServiceProvider? FunctionInvocationServices { get; } /// The logger to use for logging information about function invocation. @@ -65,15 +65,15 @@ public class FunctionInvokingRealtimeSession : DelegatingRealtimeSession private readonly ActivitySource? _activitySource; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The underlying , or the next instance in a chain of sessions. /// An to use for logging information about function invocation. /// An optional to use for resolving services required by the instances being invoked. - public FunctionInvokingRealtimeSession(IRealtimeClientSession innerSession, ILoggerFactory? loggerFactory = null, IServiceProvider? functionInvocationServices = null) + public FunctionInvokingRealtimeClientSession(IRealtimeClientSession innerSession, ILoggerFactory? loggerFactory = null, IServiceProvider? functionInvocationServices = null) : base(innerSession) { - _logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; + _logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; _activitySource = innerSession.GetService(); FunctionInvocationServices = functionInvocationServices; } @@ -150,7 +150,7 @@ public static FunctionInvocationContext? CurrentContext /// /// /// - /// Each streaming request to this might end up making + /// Each streaming request to this might end up making /// multiple function call invocations. Each time the inner session responds with /// a function call request, this session might perform that invocation and send the results /// back to the inner session. This property limits the number of times @@ -184,7 +184,7 @@ public int MaximumIterationsPerRequest /// /// /// - /// When function invocations fail with an exception, the + /// When function invocations fail with an exception, the /// continues to send responses to the inner session, optionally supplying exception information (as /// controlled by ). This allows the to /// recover from errors by trying other function parameters that might succeed. @@ -211,7 +211,7 @@ public int MaximumConsecutiveErrorsPerRequest /// Gets or sets a collection of additional tools the session is able to invoke. /// - /// These will not impact the requests sent by the , which will pass through the + /// These will not impact the requests sent by the , which will pass through the /// unmodified. However, if the inner session requests the invocation of a tool /// that was not in , this collection will also be consulted /// to look for a corresponding tool to invoke. This is useful when the service might have been preconfigured to be aware @@ -222,22 +222,22 @@ public int MaximumConsecutiveErrorsPerRequest /// Gets or sets a value indicating whether a request to call an unknown function should terminate the function calling loop. /// /// to terminate the function calling loop and return the response if a request to call a tool - /// that isn't available to the is received; to create and send a + /// that isn't available to the is received; to create and send a /// function result message to the inner session stating that the tool couldn't be found. The default is . /// /// /// - /// When , call requests to any tools that aren't available to the + /// When , call requests to any tools that aren't available to the /// will result in a response message automatically being created and returned to the inner session stating that the tool couldn't be /// found. This behavior can help in cases where a model hallucinates a function, but it's problematic if the model has been made aware /// of the existence of tools outside of the normal mechanisms, and requests one of those. can be used /// to help with that. But if instead the consumer wants to know about all function call requests that the session can't handle, /// can be set to . Upon receiving a request to call a function - /// that the doesn't know about, it will terminate the function calling loop and return + /// that the doesn't know about, it will terminate the function calling loop and return /// the response, leaving the handling of the function call requests to the consumer of the session. /// /// - /// s that the is aware of (for example, because they're in + /// s that the is aware of (for example, because they're in /// or ) but that aren't s aren't considered /// unknown, just not invocable. Any requests to a non-invocable tool will also result in the function calling loop terminating, /// regardless of . diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSessionBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSessionBuilderExtensions.cs similarity index 64% rename from src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSessionBuilderExtensions.cs rename to src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSessionBuilderExtensions.cs index 9cf7104115c..ea7291080e2 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeSessionBuilderExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSessionBuilderExtensions.cs @@ -10,24 +10,24 @@ namespace Microsoft.Extensions.AI; /// -/// Provides extension methods for attaching a to a realtime session pipeline. +/// Provides extension methods for attaching a to a realtime session pipeline. /// [Experimental("MEAI001")] -public static class FunctionInvokingRealtimeSessionBuilderExtensions +public static class FunctionInvokingRealtimeClientSessionBuilderExtensions { /// /// Enables automatic function call invocation on the realtime session pipeline. /// - /// This works by adding an instance of with default options. - /// The being used to build the realtime session pipeline. + /// This works by adding an instance of with default options. + /// The being used to build the realtime session pipeline. /// An optional to use to create a logger for logging function invocations. - /// An optional callback that can be used to configure the instance. + /// An optional callback that can be used to configure the instance. /// The supplied . /// is . - public static RealtimeSessionBuilder UseFunctionInvocation( - this RealtimeSessionBuilder builder, + public static RealtimeClientSessionBuilder UseFunctionInvocation( + this RealtimeClientSessionBuilder builder, ILoggerFactory? loggerFactory = null, - Action? configure = null) + Action? configure = null) { _ = Throw.IfNull(builder); @@ -35,7 +35,7 @@ public static RealtimeSessionBuilder UseFunctionInvocation( { loggerFactory ??= services.GetService(); - var realtimeSession = new FunctionInvokingRealtimeSession(innerSession, loggerFactory, services); + var realtimeSession = new FunctionInvokingRealtimeClientSession(innerSession, loggerFactory, services); configure?.Invoke(realtimeSession); return realtimeSession; }); diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClientSession.cs similarity index 97% rename from src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs rename to src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClientSession.cs index 7f379606062..3b801c54c64 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClientSession.cs @@ -28,7 +28,7 @@ namespace Microsoft.Extensions.AI; /// /// [Experimental("MEAI001")] -public partial class LoggingRealtimeSession : DelegatingRealtimeSession +public partial class LoggingRealtimeClientSession : DelegatingRealtimeClientSession { /// An instance used for all logging. private readonly ILogger _logger; @@ -36,10 +36,10 @@ public partial class LoggingRealtimeSession : DelegatingRealtimeSession /// The to use for serialization of state written to the logger. private JsonSerializerOptions _jsonSerializerOptions; - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// The underlying . /// An instance that will be used for all logging. - public LoggingRealtimeSession(IRealtimeClientSession innerSession, ILogger logger) + public LoggingRealtimeClientSession(IRealtimeClientSession innerSession, ILogger logger) : base(innerSession) { _logger = Throw.IfNull(logger); diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSessionBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClientSessionBuilderExtensions.cs similarity index 76% rename from src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSessionBuilderExtensions.cs rename to src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClientSessionBuilderExtensions.cs index dd6ba08d75e..f22dbcbfbe3 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeSessionBuilderExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClientSessionBuilderExtensions.cs @@ -10,17 +10,17 @@ namespace Microsoft.Extensions.AI; -/// Provides extensions for configuring instances. +/// Provides extensions for configuring instances. [Experimental("MEAI001")] -public static class LoggingRealtimeSessionBuilderExtensions +public static class LoggingRealtimeClientSessionBuilderExtensions { /// Adds logging to the realtime session pipeline. - /// The . + /// The . /// /// An optional used to create a logger with which logging should be performed. /// If not supplied, a required instance will be resolved from the service provider. /// - /// An optional callback that can be used to configure the instance. + /// An optional callback that can be used to configure the instance. /// The . /// is . /// @@ -31,10 +31,10 @@ public static class LoggingRealtimeSessionBuilderExtensions /// Messages and options are not logged at other logging levels. /// /// - public static RealtimeSessionBuilder UseLogging( - this RealtimeSessionBuilder builder, + public static RealtimeClientSessionBuilder UseLogging( + this RealtimeClientSessionBuilder builder, ILoggerFactory? loggerFactory = null, - Action? configure = null) + Action? configure = null) { _ = Throw.IfNull(builder); @@ -42,14 +42,14 @@ public static RealtimeSessionBuilder UseLogging( { loggerFactory ??= services.GetRequiredService(); - // If the factory we resolve is for the null logger, the LoggingRealtimeSession will end up + // If the factory we resolve is for the null logger, the LoggingRealtimeClientSession will end up // being an expensive nop, so skip adding it and just return the inner session. if (loggerFactory == NullLoggerFactory.Instance) { return innerSession; } - var session = new LoggingRealtimeSession(innerSession, loggerFactory.CreateLogger(typeof(LoggingRealtimeSession))); + var session = new LoggingRealtimeClientSession(innerSession, loggerFactory.CreateLogger(typeof(LoggingRealtimeClientSession))); configure?.Invoke(session); return session; }); diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs similarity index 99% rename from src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs rename to src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs index f44698a1381..1b54bd76562 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs @@ -76,7 +76,7 @@ namespace Microsoft.Extensions.AI; /// /// [Experimental("MEAI001")] -public sealed partial class OpenTelemetryRealtimeSession : DelegatingRealtimeSession +public sealed partial class OpenTelemetryRealtimeClientSession : DelegatingRealtimeClientSession { private readonly ActivitySource _activitySource; private readonly Meter _meter; @@ -91,12 +91,12 @@ public sealed partial class OpenTelemetryRealtimeSession : DelegatingRealtimeSes private JsonSerializerOptions _jsonSerializerOptions; - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// The underlying . /// The to use for emitting any logging data from the session. /// An optional source name that will be used on the telemetry data. #pragma warning disable IDE0060 // Remove unused parameter; it exists for backwards compatibility and future use - public OpenTelemetryRealtimeSession(IRealtimeClientSession innerSession, ILogger? logger = null, string? sourceName = null) + public OpenTelemetryRealtimeClientSession(IRealtimeClientSession innerSession, ILogger? logger = null, string? sourceName = null) #pragma warning restore IDE0060 : base(innerSession) { diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSessionBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSessionBuilderExtensions.cs similarity index 84% rename from src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSessionBuilderExtensions.cs rename to src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSessionBuilderExtensions.cs index e0666112c06..eb69e12c1ac 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeSessionBuilderExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSessionBuilderExtensions.cs @@ -9,9 +9,9 @@ namespace Microsoft.Extensions.AI; -/// Provides extensions for configuring instances. +/// Provides extensions for configuring instances. [Experimental("MEAI001")] -public static class OpenTelemetryRealtimeSessionBuilderExtensions +public static class OpenTelemetryRealtimeClientSessionBuilderExtensions { /// /// Adds OpenTelemetry support to the realtime session pipeline, following the OpenTelemetry Semantic Conventions for Generative AI systems. @@ -55,22 +55,22 @@ public static class OpenTelemetryRealtimeSessionBuilderExtensions /// /// /// - /// The . + /// The . /// An optional to use to create a logger for logging events. /// An optional source name that will be used on the telemetry data. - /// An optional callback that can be used to configure the instance. + /// An optional callback that can be used to configure the instance. /// The . /// is . - public static RealtimeSessionBuilder UseOpenTelemetry( - this RealtimeSessionBuilder builder, + public static RealtimeClientSessionBuilder UseOpenTelemetry( + this RealtimeClientSessionBuilder builder, ILoggerFactory? loggerFactory = null, string? sourceName = null, - Action? configure = null) => + Action? configure = null) => Throw.IfNull(builder).Use((innerSession, services) => { loggerFactory ??= services.GetService(); - var session = new OpenTelemetryRealtimeSession(innerSession, loggerFactory?.CreateLogger(typeof(OpenTelemetryRealtimeSession)), sourceName); + var session = new OpenTelemetryRealtimeClientSession(innerSession, loggerFactory?.CreateLogger(typeof(OpenTelemetryRealtimeClientSession)), sourceName); configure?.Invoke(session); return session; diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilder.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionBuilder.cs similarity index 82% rename from src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilder.cs rename to src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionBuilder.cs index 3240c11c0c4..2711e09c0bf 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilder.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionBuilder.cs @@ -11,25 +11,25 @@ namespace Microsoft.Extensions.AI; /// A builder for creating pipelines of . [Experimental("MEAI001")] -public sealed class RealtimeSessionBuilder +public sealed class RealtimeClientSessionBuilder { private readonly Func _innerSessionFactory; /// The registered session factory instances. private List>? _sessionFactories; - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// The inner that represents the underlying backend. /// is . - public RealtimeSessionBuilder(IRealtimeClientSession innerSession) + public RealtimeClientSessionBuilder(IRealtimeClientSession innerSession) { _ = Throw.IfNull(innerSession); _innerSessionFactory = _ => innerSession; } - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// A callback that produces the inner that represents the underlying backend. - public RealtimeSessionBuilder(Func innerSessionFactory) + public RealtimeClientSessionBuilder(Func innerSessionFactory) { _innerSessionFactory = Throw.IfNull(innerSessionFactory); } @@ -54,7 +54,7 @@ public IRealtimeClientSession Build(IServiceProvider? services = null) if (session is null) { Throw.InvalidOperationException( - $"The {nameof(RealtimeSessionBuilder)} entry at index {i} returned null. " + + $"The {nameof(RealtimeClientSessionBuilder)} entry at index {i} returned null. " + $"Ensure that the callbacks passed to {nameof(Use)} return non-null {nameof(IRealtimeClientSession)} instances."); } } @@ -65,9 +65,9 @@ public IRealtimeClientSession Build(IServiceProvider? services = null) /// Adds a factory for an intermediate realtime session to the realtime session pipeline. /// The session factory function. - /// The updated instance. + /// The updated instance. /// is . - public RealtimeSessionBuilder Use(Func sessionFactory) + public RealtimeClientSessionBuilder Use(Func sessionFactory) { _ = Throw.IfNull(sessionFactory); @@ -76,9 +76,9 @@ public RealtimeSessionBuilder Use(FuncAdds a factory for an intermediate realtime session to the realtime session pipeline. /// The session factory function. - /// The updated instance. + /// The updated instance. /// is . - public RealtimeSessionBuilder Use(Func sessionFactory) + public RealtimeClientSessionBuilder Use(Func sessionFactory) { _ = Throw.IfNull(sessionFactory); @@ -96,17 +96,17 @@ public RealtimeSessionBuilder Use(Func - /// The updated instance. + /// The updated instance. /// /// This overload can be used when the anonymous implementation needs to provide pre-processing and/or post-processing /// for the streaming response. /// /// is . - public RealtimeSessionBuilder Use( + public RealtimeClientSessionBuilder Use( Func> getStreamingResponseFunc) { _ = Throw.IfNull(getStreamingResponseFunc); - return Use((innerSession, _) => new AnonymousDelegatingRealtimeSession(innerSession, getStreamingResponseFunc)); + return Use((innerSession, _) => new AnonymousDelegatingRealtimeClientSession(innerSession, getStreamingResponseFunc)); } } diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilderRealtimeSessionExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionBuilderRealtimeClientSessionExtensions.cs similarity index 60% rename from src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilderRealtimeSessionExtensions.cs rename to src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionBuilderRealtimeClientSessionExtensions.cs index 390043243c9..84def11121d 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionBuilderRealtimeSessionExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionBuilderRealtimeClientSessionExtensions.cs @@ -7,22 +7,22 @@ namespace Microsoft.Extensions.AI; -/// Provides extension methods for working with in the context of . +/// Provides extension methods for working with in the context of . [Experimental("MEAI001")] -public static class RealtimeSessionBuilderRealtimeSessionExtensions +public static class RealtimeClientSessionBuilderRealtimeClientSessionExtensions { - /// Creates a new using as its inner session. + /// Creates a new using as its inner session. /// The session to use as the inner session. - /// The new instance. + /// The new instance. /// - /// This method is equivalent to using the constructor directly, + /// This method is equivalent to using the constructor directly, /// specifying as the inner session. /// /// is . - public static RealtimeSessionBuilder AsBuilder(this IRealtimeClientSession innerSession) + public static RealtimeClientSessionBuilder AsBuilder(this IRealtimeClientSession innerSession) { _ = Throw.IfNull(innerSession); - return new RealtimeSessionBuilder(innerSession); + return new RealtimeClientSessionBuilder(innerSession); } } diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionExtensions.cs similarity index 96% rename from src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionExtensions.cs rename to src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionExtensions.cs index 0063b0d5c0d..c9a074c78e5 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeSessionExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionExtensions.cs @@ -9,7 +9,7 @@ namespace Microsoft.Extensions.AI; /// Provides a collection of static methods for extending instances. [Experimental("MEAI001")] -public static class RealtimeSessionExtensions +public static class RealtimeClientSessionExtensions { /// Asks the for an object of type . /// The type of the object to be retrieved. diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeClientSession.cs similarity index 96% rename from test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs rename to test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeClientSession.cs index 82fccc46230..a7bbb399c8b 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeSession.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestRealtimeClientSession.cs @@ -9,7 +9,7 @@ namespace Microsoft.Extensions.AI; /// A test implementation that uses callbacks for verification. -public sealed class TestRealtimeSession : IRealtimeClientSession +public sealed class TestRealtimeClientSession : IRealtimeClientSession { /// Gets or sets the callback to invoke when is called. public Func? SendAsyncCallback { get; set; } diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientSessionTests.cs similarity index 77% rename from test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs rename to test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientSessionTests.cs index a6188b8fdfc..c2037a7ff21 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientSessionTests.cs @@ -10,30 +10,30 @@ namespace Microsoft.Extensions.AI; -public class OpenAIRealtimeSessionTests +public class OpenAIRealtimeClientSessionTests { [Fact] public async Task GetService_ReturnsExpectedServices() { - await using IRealtimeClientSession session = new OpenAIRealtimeSession("key", "model"); + await using IRealtimeClientSession session = new OpenAIRealtimeClientSession("key", "model"); - Assert.Same(session, session.GetService(typeof(OpenAIRealtimeSession))); + Assert.Same(session, session.GetService(typeof(OpenAIRealtimeClientSession))); Assert.Same(session, session.GetService(typeof(IRealtimeClientSession))); Assert.Null(session.GetService(typeof(string))); - Assert.Null(session.GetService(typeof(OpenAIRealtimeSession), "someKey")); + Assert.Null(session.GetService(typeof(OpenAIRealtimeClientSession), "someKey")); } [Fact] public async Task GetService_NullServiceType_Throws() { - await using IRealtimeClientSession session = new OpenAIRealtimeSession("key", "model"); + await using IRealtimeClientSession session = new OpenAIRealtimeClientSession("key", "model"); Assert.Throws("serviceType", () => session.GetService(null!)); } [Fact] public async Task DisposeAsync_CanBeCalledMultipleTimes() { - IRealtimeClientSession session = new OpenAIRealtimeSession("key", "model"); + IRealtimeClientSession session = new OpenAIRealtimeClientSession("key", "model"); await session.DisposeAsync(); // Second dispose should not throw. @@ -44,7 +44,7 @@ public async Task DisposeAsync_CanBeCalledMultipleTimes() [Fact] public async Task Options_InitiallyNull() { - await using var session = new OpenAIRealtimeSession("key", "model"); + await using var session = new OpenAIRealtimeClientSession("key", "model"); Assert.Null(session.Options); } @@ -57,14 +57,14 @@ public void SessionUpdateMessage_NullOptions_Throws() [Fact] public async Task SendAsync_NullMessage_Throws() { - await using var session = new OpenAIRealtimeSession("key", "model"); + await using var session = new OpenAIRealtimeClientSession("key", "model"); await Assert.ThrowsAsync("message", () => session.SendAsync(null!)); } [Fact] public async Task SendAsync_CancelledToken_ReturnsSilently() { - await using var session = new OpenAIRealtimeSession("key", "model"); + await using var session = new OpenAIRealtimeClientSession("key", "model"); using var cts = new CancellationTokenSource(); cts.Cancel(); @@ -76,7 +76,7 @@ public async Task SendAsync_CancelledToken_ReturnsSilently() [Fact] public async Task ConnectAsync_CancelledToken_Throws() { - await using var session = new OpenAIRealtimeSession("key", "model"); + await using var session = new OpenAIRealtimeClientSession("key", "model"); using var cts = new CancellationTokenSource(); cts.Cancel(); diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Microsoft.Extensions.AI.Tests.csproj b/test/Libraries/Microsoft.Extensions.AI.Tests/Microsoft.Extensions.AI.Tests.csproj index c042d22f17f..29c93c40f81 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Microsoft.Extensions.AI.Tests.csproj +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Microsoft.Extensions.AI.Tests.csproj @@ -24,7 +24,7 @@ - + diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeClientSessionTests.cs similarity index 71% rename from test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs rename to test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeClientSessionTests.cs index c2c950c0951..3ab43404473 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeClientSessionTests.cs @@ -12,20 +12,20 @@ namespace Microsoft.Extensions.AI; -public class DelegatingRealtimeSessionTests +public class DelegatingRealtimeClientSessionTests { [Fact] public void Ctor_NullInnerSession_Throws() { - Assert.Throws("innerSession", () => new NoOpDelegatingRealtimeSession(null!)); + Assert.Throws("innerSession", () => new NoOpDelegatingRealtimeClientSession(null!)); } [Fact] public async Task Options_DelegatesToInner() { var expectedOptions = new RealtimeSessionOptions { Model = "test-model" }; - await using var inner = new TestRealtimeSession { Options = expectedOptions }; - await using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var inner = new TestRealtimeClientSession { Options = expectedOptions }; + await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); Assert.Same(expectedOptions, delegating.Options); } @@ -35,7 +35,7 @@ public async Task SendAsync_SessionUpdateMessage_DelegatesToInner() { var called = false; var sentOptions = new RealtimeSessionOptions { Instructions = "Be helpful" }; - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { SendAsyncCallback = (msg, _) => { @@ -45,7 +45,7 @@ public async Task SendAsync_SessionUpdateMessage_DelegatesToInner() return Task.CompletedTask; }, }; - await using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); await delegating.SendAsync(new RealtimeClientSessionUpdateMessage(sentOptions)); Assert.True(called); @@ -56,7 +56,7 @@ public async Task SendAsync_DelegatesToInner() { var called = false; var sentMessage = new RealtimeClientMessage { MessageId = "evt_001" }; - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { SendAsyncCallback = (msg, _) => { @@ -65,7 +65,7 @@ public async Task SendAsync_DelegatesToInner() return Task.CompletedTask; }, }; - await using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); await delegating.SendAsync(sentMessage); Assert.True(called); @@ -75,11 +75,11 @@ public async Task SendAsync_DelegatesToInner() public async Task GetStreamingResponseAsync_DelegatesToInner() { var expected = new RealtimeServerMessage { Type = RealtimeServerMessageType.Error, MessageId = "evt_002" }; - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { GetStreamingResponseAsyncCallback = (ct) => YieldSingle(expected, ct), }; - await using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); var messages = new List(); await foreach (var msg in delegating.GetStreamingResponseAsync()) @@ -94,40 +94,40 @@ public async Task GetStreamingResponseAsync_DelegatesToInner() [Fact] public async Task GetService_ReturnsSelfForMatchingType() { - await using var inner = new TestRealtimeSession(); - await using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var inner = new TestRealtimeClientSession(); + await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); - Assert.Same(delegating, delegating.GetService(typeof(NoOpDelegatingRealtimeSession))); - Assert.Same(delegating, delegating.GetService(typeof(DelegatingRealtimeSession))); + Assert.Same(delegating, delegating.GetService(typeof(NoOpDelegatingRealtimeClientSession))); + Assert.Same(delegating, delegating.GetService(typeof(DelegatingRealtimeClientSession))); Assert.Same(delegating, delegating.GetService(typeof(IRealtimeClientSession))); } [Fact] public async Task GetService_DelegatesToInnerForUnknownType() { - await using var inner = new TestRealtimeSession(); - await using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var inner = new TestRealtimeClientSession(); + await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); - // TestRealtimeSession returns itself for matching types - Assert.Same(inner, delegating.GetService(typeof(TestRealtimeSession))); + // TestRealtimeClientSession returns itself for matching types + Assert.Same(inner, delegating.GetService(typeof(TestRealtimeClientSession))); Assert.Null(delegating.GetService(typeof(string))); } [Fact] public async Task GetService_WithServiceKey_DelegatesToInner() { - await using var inner = new TestRealtimeSession(); - await using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var inner = new TestRealtimeClientSession(); + await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); // With a non-null key, delegating should NOT return itself even for matching types - Assert.Null(delegating.GetService(typeof(NoOpDelegatingRealtimeSession), "someKey")); + Assert.Null(delegating.GetService(typeof(NoOpDelegatingRealtimeClientSession), "someKey")); } [Fact] public async Task GetService_NullServiceType_Throws() { - await using var inner = new TestRealtimeSession(); - await using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var inner = new TestRealtimeClientSession(); + await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); Assert.Throws("serviceType", () => delegating.GetService(null!)); } @@ -136,8 +136,8 @@ public async Task GetService_NullServiceType_Throws() public async Task DisposeAsync_DisposesInner() { var disposed = false; - await using var inner = new DisposableTestRealtimeSession(() => disposed = true); - var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var inner = new DisposableTestRealtimeClientSession(() => disposed = true); + var delegating = new NoOpDelegatingRealtimeClientSession(inner); await delegating.DisposeAsync(); Assert.True(disposed); @@ -150,7 +150,7 @@ public async Task SendAsync_SessionUpdateMessage_FlowsCancellationToken() using var cts = new CancellationTokenSource(); var sentOptions = new RealtimeSessionOptions(); - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { SendAsyncCallback = (msg, ct) => { @@ -158,7 +158,7 @@ public async Task SendAsync_SessionUpdateMessage_FlowsCancellationToken() return Task.CompletedTask; }, }; - await using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); await delegating.SendAsync(new RealtimeClientSessionUpdateMessage(sentOptions), cts.Token); Assert.Equal(cts.Token, capturedToken); @@ -171,7 +171,7 @@ public async Task SendAsync_FlowsCancellationToken() using var cts = new CancellationTokenSource(); var sentMessage = new RealtimeClientMessage(); - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { SendAsyncCallback = (msg, ct) => { @@ -179,7 +179,7 @@ public async Task SendAsync_FlowsCancellationToken() return Task.CompletedTask; }, }; - await using var delegating = new NoOpDelegatingRealtimeSession(inner); + await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); await delegating.SendAsync(sentMessage, cts.Token); Assert.Equal(cts.Token, capturedToken); @@ -194,21 +194,21 @@ private static async IAsyncEnumerable YieldSingle( yield return message; } - /// A concrete DelegatingRealtimeSession for testing (since the base class is abstract-ish with protected ctor). - private sealed class NoOpDelegatingRealtimeSession : DelegatingRealtimeSession + /// A concrete DelegatingRealtimeClientSession for testing (since the base class is abstract-ish with protected ctor). + private sealed class NoOpDelegatingRealtimeClientSession : DelegatingRealtimeClientSession { - public NoOpDelegatingRealtimeSession(IRealtimeClientSession innerSession) + public NoOpDelegatingRealtimeClientSession(IRealtimeClientSession innerSession) : base(innerSession) { } } /// A test session that tracks Dispose calls. - private sealed class DisposableTestRealtimeSession : IRealtimeClientSession + private sealed class DisposableTestRealtimeClientSession : IRealtimeClientSession { private readonly Action _onDispose; - public DisposableTestRealtimeSession(Action onDispose) + public DisposableTestRealtimeClientSession(Action onDispose) { _onDispose = onDispose; } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeClientSessionTests.cs similarity index 88% rename from test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs rename to test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeClientSessionTests.cs index a3924ae507a..0020a09f794 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeClientSessionTests.cs @@ -15,19 +15,19 @@ namespace Microsoft.Extensions.AI; -public class FunctionInvokingRealtimeSessionTests +public class FunctionInvokingRealtimeClientSessionTests { [Fact] public void Ctor_NullInnerSession_Throws() { - Assert.Throws("innerSession", () => new FunctionInvokingRealtimeSession(null!)); + Assert.Throws("innerSession", () => new FunctionInvokingRealtimeClientSession(null!)); } [Fact] public async Task Properties_DefaultValues() { - await using var inner = new TestRealtimeSession(); - await using var session = new FunctionInvokingRealtimeSession(inner); + await using var inner = new TestRealtimeClientSession(); + await using var session = new FunctionInvokingRealtimeClientSession(inner); Assert.False(session.IncludeDetailedErrors); Assert.False(session.AllowConcurrentInvocation); @@ -41,8 +41,8 @@ public async Task Properties_DefaultValues() [Fact] public async Task MaximumIterationsPerRequest_InvalidValue_Throws() { - await using var inner = new TestRealtimeSession(); - await using var session = new FunctionInvokingRealtimeSession(inner); + await using var inner = new TestRealtimeClientSession(); + await using var session = new FunctionInvokingRealtimeClientSession(inner); Assert.Throws("value", () => session.MaximumIterationsPerRequest = 0); Assert.Throws("value", () => session.MaximumIterationsPerRequest = -1); @@ -51,8 +51,8 @@ public async Task MaximumIterationsPerRequest_InvalidValue_Throws() [Fact] public async Task MaximumConsecutiveErrorsPerRequest_InvalidValue_Throws() { - await using var inner = new TestRealtimeSession(); - await using var session = new FunctionInvokingRealtimeSession(inner); + await using var inner = new TestRealtimeClientSession(); + await using var session = new FunctionInvokingRealtimeClientSession(inner); Assert.Throws("value", () => session.MaximumConsecutiveErrorsPerRequest = -1); @@ -70,11 +70,11 @@ public async Task GetStreamingResponseAsync_NoFunctionCalls_PassesThrough() new() { Type = RealtimeServerMessageType.ResponseDone, MessageId = "evt_002" }, }; - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { GetStreamingResponseAsyncCallback = (ct) => YieldMessages(serverMessages, ct), }; - await using var session = new FunctionInvokingRealtimeSession(inner); + await using var session = new FunctionInvokingRealtimeClientSession(inner); var received = new List(); await foreach (var msg in session.GetStreamingResponseAsync()) @@ -96,7 +96,7 @@ public async Task GetStreamingResponseAsync_FunctionCall_InvokesAndInjectsResult "Gets the weather"); var injectedMessages = new List(); - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Tools = [getWeather] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages( @@ -110,7 +110,7 @@ public async Task GetStreamingResponseAsync_FunctionCall_InvokesAndInjectsResult }, }; - await using var session = new FunctionInvokingRealtimeSession(inner); + await using var session = new FunctionInvokingRealtimeClientSession(inner); var received = new List(); await foreach (var msg in session.GetStreamingResponseAsync()) @@ -145,7 +145,7 @@ public async Task GetStreamingResponseAsync_FunctionCall_FromAdditionalTools() "Gets weather"); var injectedMessages = new List(); - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ @@ -158,7 +158,7 @@ public async Task GetStreamingResponseAsync_FunctionCall_FromAdditionalTools() }, }; - await using var session = new FunctionInvokingRealtimeSession(inner) + await using var session = new FunctionInvokingRealtimeClientSession(inner) { AdditionalTools = [getWeather], }; @@ -190,14 +190,14 @@ public async Task GetStreamingResponseAsync_MaxIterations_StopsInvoking() var messages = Enumerable.Range(0, 5).Select(i => CreateFunctionCallOutputItemMessage($"call_{i}", "counter", null)).ToList(); - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Tools = [countFunc] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages(messages, ct), SendAsyncCallback = (_, _) => Task.CompletedTask, }; - await using var session = new FunctionInvokingRealtimeSession(inner) + await using var session = new FunctionInvokingRealtimeClientSession(inner) { MaximumIterationsPerRequest = 2, }; @@ -224,7 +224,7 @@ public async Task GetStreamingResponseAsync_FunctionInvoker_CustomDelegate() "my_func", "Test"); - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Tools = [myFunc] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages( @@ -234,7 +234,7 @@ public async Task GetStreamingResponseAsync_FunctionInvoker_CustomDelegate() SendAsyncCallback = (_, _) => Task.CompletedTask, }; - await using var session = new FunctionInvokingRealtimeSession(inner) + await using var session = new FunctionInvokingRealtimeClientSession(inner) { FunctionInvoker = (context, ct) => { @@ -255,7 +255,7 @@ public async Task GetStreamingResponseAsync_FunctionInvoker_CustomDelegate() public async Task GetStreamingResponseAsync_UnknownFunction_SendsErrorByDefault() { var injectedMessages = new List(); - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ @@ -268,7 +268,7 @@ public async Task GetStreamingResponseAsync_UnknownFunction_SendsErrorByDefault( }, }; - await using var session = new FunctionInvokingRealtimeSession(inner); + await using var session = new FunctionInvokingRealtimeClientSession(inner); await foreach (var msg in session.GetStreamingResponseAsync()) { @@ -291,7 +291,7 @@ public async Task GetStreamingResponseAsync_FunctionError_IncludesDetailedErrors "Fails"); var injectedMessages = new List(); - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Tools = [failFunc] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages( @@ -305,7 +305,7 @@ public async Task GetStreamingResponseAsync_FunctionError_IncludesDetailedErrors }, }; - await using var session = new FunctionInvokingRealtimeSession(inner) + await using var session = new FunctionInvokingRealtimeClientSession(inner) { IncludeDetailedErrors = true, }; @@ -330,7 +330,7 @@ public async Task GetStreamingResponseAsync_FunctionError_HidesDetailsWhenNotEna "Fails"); var injectedMessages = new List(); - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Tools = [failFunc] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages( @@ -344,7 +344,7 @@ public async Task GetStreamingResponseAsync_FunctionError_HidesDetailsWhenNotEna }, }; - await using var session = new FunctionInvokingRealtimeSession(inner) + await using var session = new FunctionInvokingRealtimeClientSession(inner) { IncludeDetailedErrors = false, }; @@ -363,19 +363,19 @@ public async Task GetStreamingResponseAsync_FunctionError_HidesDetailsWhenNotEna [Fact] public async Task GetService_ReturnsSelf() { - await using var inner = new TestRealtimeSession(); - await using var session = new FunctionInvokingRealtimeSession(inner); + await using var inner = new TestRealtimeClientSession(); + await using var session = new FunctionInvokingRealtimeClientSession(inner); - Assert.Same(session, session.GetService(typeof(FunctionInvokingRealtimeSession))); + Assert.Same(session, session.GetService(typeof(FunctionInvokingRealtimeClientSession))); Assert.Same(session, session.GetService(typeof(IRealtimeClientSession))); - Assert.Same(inner, session.GetService(typeof(TestRealtimeSession))); + Assert.Same(inner, session.GetService(typeof(TestRealtimeClientSession))); } [Fact] public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_StopsLoop() { var injectedMessages = new List(); - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ @@ -389,7 +389,7 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_StopsLoop() }, }; - await using var session = new FunctionInvokingRealtimeSession(inner) + await using var session = new FunctionInvokingRealtimeClientSession(inner) { TerminateOnUnknownCalls = true, }; @@ -411,7 +411,7 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_StopsLoop() public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_False_SendsError() { var injectedMessages = new List(); - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { GetStreamingResponseAsyncCallback = (ct) => YieldMessages( [ @@ -424,7 +424,7 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_False_SendsE }, }; - await using var session = new FunctionInvokingRealtimeSession(inner) + await using var session = new FunctionInvokingRealtimeClientSession(inner) { TerminateOnUnknownCalls = false, }; @@ -495,14 +495,14 @@ public async Task GetStreamingResponseAsync_ConcurrentInvocation_InvokesInParall Item = combinedItem, }; - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Tools = [slowFunc] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages([combinedMessage], ct), SendAsyncCallback = (_, _) => Task.CompletedTask, }; - await using var session = new FunctionInvokingRealtimeSession(inner) + await using var session = new FunctionInvokingRealtimeClientSession(inner) { AllowConcurrentInvocation = true, }; @@ -538,14 +538,14 @@ public async Task GetStreamingResponseAsync_ConsecutiveErrors_ExceedsLimit_Throw CreateFunctionCallOutputItemMessage("call_4", "fail_func", null), }; - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Tools = [failFunc] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages(messages, ct), SendAsyncCallback = (_, _) => Task.CompletedTask, }; - await using var session = new FunctionInvokingRealtimeSession(inner) + await using var session = new FunctionInvokingRealtimeClientSession(inner) { MaximumConsecutiveErrorsPerRequest = 1, }; @@ -564,14 +564,14 @@ await Assert.ThrowsAsync(async () => public void UseFunctionInvocation_NullBuilder_Throws() { Assert.Throws("builder", () => - ((RealtimeSessionBuilder)null!).UseFunctionInvocation()); + ((RealtimeClientSessionBuilder)null!).UseFunctionInvocation()); } [Fact] public async Task UseFunctionInvocation_ConfigureCallback_IsInvoked() { - await using var inner = new TestRealtimeSession(); - var builder = new RealtimeSessionBuilder(inner); + await using var inner = new TestRealtimeClientSession(); + var builder = new RealtimeClientSessionBuilder(inner); bool configured = false; builder.UseFunctionInvocation(configure: session => @@ -584,10 +584,10 @@ public async Task UseFunctionInvocation_ConfigureCallback_IsInvoked() await using var pipeline = builder.Build(); Assert.True(configured); - var funcSession = pipeline.GetService(typeof(FunctionInvokingRealtimeSession)); + var funcSession = pipeline.GetService(typeof(FunctionInvokingRealtimeClientSession)); Assert.NotNull(funcSession); - var typedSession = Assert.IsType(funcSession); + var typedSession = Assert.IsType(funcSession); Assert.True(typedSession.IncludeDetailedErrors); Assert.Equal(10, typedSession.MaximumIterationsPerRequest); } @@ -600,7 +600,7 @@ public async Task GetStreamingResponseAsync_NonInvocableTool_TerminatesLoop() var declaration = AIFunctionFactory.CreateDeclaration("my_declaration", "A non-invocable tool", schema); var injectedMessages = new List(); - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Tools = [declaration] }, GetStreamingResponseAsyncCallback = (ct) => YieldMessages( @@ -615,7 +615,7 @@ public async Task GetStreamingResponseAsync_NonInvocableTool_TerminatesLoop() }, }; - await using var session = new FunctionInvokingRealtimeSession(inner); + await using var session = new FunctionInvokingRealtimeClientSession(inner); var received = new List(); await foreach (var msg in session.GetStreamingResponseAsync()) diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeClientSessionTests.cs similarity index 88% rename from test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs rename to test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeClientSessionTests.cs index a46543064c4..c8c8d895c54 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeClientSessionTests.cs @@ -14,33 +14,33 @@ namespace Microsoft.Extensions.AI; -public class LoggingRealtimeSessionTests +public class LoggingRealtimeClientSessionTests { [Fact] - public async Task LoggingRealtimeSession_InvalidArgs_Throws() + public async Task LoggingRealtimeClientSession_InvalidArgs_Throws() { - await using var innerSession = new TestRealtimeSession(); - Assert.Throws("innerSession", () => new LoggingRealtimeSession(null!, NullLogger.Instance)); - Assert.Throws("logger", () => new LoggingRealtimeSession(innerSession, null!)); + await using var innerSession = new TestRealtimeClientSession(); + Assert.Throws("innerSession", () => new LoggingRealtimeClientSession(null!, NullLogger.Instance)); + Assert.Throws("logger", () => new LoggingRealtimeClientSession(innerSession, null!)); } [Fact] public async Task UseLogging_AvoidsInjectingNopSession() { - await using var innerSession = new TestRealtimeSession(); + await using var innerSession = new TestRealtimeClientSession(); - Assert.Null(innerSession.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build().GetService(typeof(LoggingRealtimeSession))); + Assert.Null(innerSession.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build().GetService(typeof(LoggingRealtimeClientSession))); Assert.Same(innerSession, innerSession.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build().GetService(typeof(IRealtimeClientSession))); using var factory = LoggerFactory.Create(b => b.AddFakeLogging()); - Assert.NotNull(innerSession.AsBuilder().UseLogging(factory).Build().GetService(typeof(LoggingRealtimeSession))); + Assert.NotNull(innerSession.AsBuilder().UseLogging(factory).Build().GetService(typeof(LoggingRealtimeClientSession))); ServiceCollection c = new(); c.AddFakeLogging(); var services = c.BuildServiceProvider(); - Assert.NotNull(innerSession.AsBuilder().UseLogging().Build(services).GetService(typeof(LoggingRealtimeSession))); - Assert.NotNull(innerSession.AsBuilder().UseLogging(null).Build(services).GetService(typeof(LoggingRealtimeSession))); - Assert.Null(innerSession.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build(services).GetService(typeof(LoggingRealtimeSession))); + Assert.NotNull(innerSession.AsBuilder().UseLogging().Build(services).GetService(typeof(LoggingRealtimeClientSession))); + Assert.NotNull(innerSession.AsBuilder().UseLogging(null).Build(services).GetService(typeof(LoggingRealtimeClientSession))); + Assert.Null(innerSession.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build(services).GetService(typeof(LoggingRealtimeClientSession))); } [Theory] @@ -55,7 +55,7 @@ public async Task SendAsync_SessionUpdateMessage_LogsInvocationAndCompletion(Log c.AddLogging(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(level)); var services = c.BuildServiceProvider(); - await using var innerSession = new TestRealtimeSession(); + await using var innerSession = new TestRealtimeClientSession(); await using var session = innerSession .AsBuilder() @@ -95,7 +95,7 @@ public async Task SendAsync_LogsInvocationAndCompletion(LogLevel level) c.AddLogging(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(level)); var services = c.BuildServiceProvider(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { SendAsyncCallback = (message, cancellationToken) => Task.CompletedTask, }; @@ -135,7 +135,7 @@ public async Task GetStreamingResponseAsync_LogsMessagesReceived(LogLevel level) var collector = new FakeLogCollector(); using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(level)); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { GetStreamingResponseAsyncCallback = (cancellationToken) => GetMessagesAsync() }; @@ -188,7 +188,7 @@ public async Task SendAsync_SessionUpdateMessage_LogsCancellation() using var cts = new CancellationTokenSource(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { SendAsyncCallback = (msg, cancellationToken) => { @@ -216,7 +216,7 @@ public async Task SendAsync_SessionUpdateMessage_LogsErrors() var collector = new FakeLogCollector(); using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { SendAsyncCallback = (msg, cancellationToken) => { @@ -245,7 +245,7 @@ public async Task GetStreamingResponseAsync_LogsCancellation() using var cts = new CancellationTokenSource(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { GetStreamingResponseAsyncCallback = (cancellationToken) => ThrowCancellationAsync(cancellationToken) }; @@ -285,7 +285,7 @@ public async Task GetStreamingResponseAsync_LogsErrors() var collector = new FakeLogCollector(); using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { GetStreamingResponseAsyncCallback = (cancellationToken) => ThrowErrorAsync() }; @@ -323,14 +323,14 @@ public async Task GetService_ReturnsLoggingSessionWhenRequested() { using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddFakeLogging()); - await using var innerSession = new TestRealtimeSession(); + await using var innerSession = new TestRealtimeClientSession(); await using var session = innerSession .AsBuilder() .UseLogging(loggerFactory) .Build(); - Assert.NotNull(session.GetService(typeof(LoggingRealtimeSession))); + Assert.NotNull(session.GetService(typeof(LoggingRealtimeClientSession))); Assert.Same(session, session.GetService(typeof(IRealtimeClientSession))); } @@ -342,7 +342,7 @@ public async Task SendAsync_LogsCancellation() using var cts = new CancellationTokenSource(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { SendAsyncCallback = (message, cancellationToken) => { @@ -371,7 +371,7 @@ public async Task SendAsync_LogsErrors() var collector = new FakeLogCollector(); using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(LogLevel.Debug)); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { SendAsyncCallback = (message, cancellationToken) => { @@ -396,8 +396,8 @@ await Assert.ThrowsAsync(() => [Fact] public async Task JsonSerializerOptions_NullValue_Throws() { - await using var innerSession = new TestRealtimeSession(); - await using var session = new LoggingRealtimeSession(innerSession, NullLogger.Instance); + await using var innerSession = new TestRealtimeClientSession(); + await using var session = new LoggingRealtimeClientSession(innerSession, NullLogger.Instance); Assert.Throws("value", () => session.JsonSerializerOptions = null!); } @@ -405,8 +405,8 @@ public async Task JsonSerializerOptions_NullValue_Throws() [Fact] public async Task JsonSerializerOptions_Roundtrip() { - await using var innerSession = new TestRealtimeSession(); - await using var session = new LoggingRealtimeSession(innerSession, NullLogger.Instance); + await using var innerSession = new TestRealtimeClientSession(); + await using var session = new LoggingRealtimeClientSession(innerSession, NullLogger.Instance); var customOptions = new System.Text.Json.JsonSerializerOptions(); session.JsonSerializerOptions = customOptions; @@ -418,13 +418,13 @@ public async Task JsonSerializerOptions_Roundtrip() public void UseLogging_NullBuilder_Throws() { Assert.Throws("builder", () => - ((RealtimeSessionBuilder)null!).UseLogging()); + ((RealtimeClientSessionBuilder)null!).UseLogging()); } [Fact] public async Task UseLogging_ConfigureCallback_IsInvoked() { - await using var innerSession = new TestRealtimeSession(); + await using var innerSession = new TestRealtimeClientSession(); using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddFakeLogging()); bool configured = false; diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientSessionTests.cs similarity index 94% rename from test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs rename to test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientSessionTests.cs index 4f12dfba8f3..0c68e37dd50 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientSessionTests.cs @@ -14,7 +14,7 @@ namespace Microsoft.Extensions.AI; -public class OpenTelemetryRealtimeSessionTests +public class OpenTelemetryRealtimeClientSessionTests { [Theory] [InlineData(false)] @@ -28,7 +28,7 @@ public async Task ExpectedInformationLogged_GetStreamingResponseAsync(bool enabl .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { @@ -190,7 +190,7 @@ public async Task GetStreamingResponseAsync_TracesError() .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetStreamingResponseAsyncCallback = (cancellationToken) => ThrowingCallbackAsync(cancellationToken), @@ -233,7 +233,7 @@ public async Task GetStreamingResponseAsync_TracesErrorFromResponse() .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetStreamingResponseAsyncCallback = (cancellationToken) => ErrorResponseCallbackAsync(cancellationToken), @@ -281,7 +281,7 @@ public async Task NoListeners_NoActivityCreated() .AddSource("different-source") .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetStreamingResponseAsyncCallback = (cancellationToken) => EmptyCallbackAsync(cancellationToken), @@ -317,10 +317,10 @@ static async IAsyncEnumerable EmptyCallbackAsync([Enumera [Fact] public async Task InvalidArgs_Throws() { - await using var innerSession = new TestRealtimeSession(); + await using var innerSession = new TestRealtimeClientSession(); - Assert.Throws("innerSession", () => new OpenTelemetryRealtimeSession(null!)); - Assert.Throws("value", () => new OpenTelemetryRealtimeSession(innerSession).JsonSerializerOptions = null!); + Assert.Throws("innerSession", () => new OpenTelemetryRealtimeClientSession(null!)); + Assert.Throws("value", () => new OpenTelemetryRealtimeClientSession(innerSession).JsonSerializerOptions = null!); } [Fact] @@ -332,8 +332,8 @@ public void SessionUpdateMessage_NullOptions_Throws() [Fact] public async Task GetService_ReturnsActivitySource() { - await using var innerSession = new TestRealtimeSession(); - await using var session = new OpenTelemetryRealtimeSession(innerSession); + await using var innerSession = new TestRealtimeClientSession(); + await using var session = new OpenTelemetryRealtimeClientSession(innerSession); var activitySource = session.GetService(typeof(ActivitySource)); Assert.NotNull(activitySource); @@ -343,10 +343,10 @@ public async Task GetService_ReturnsActivitySource() [Fact] public async Task GetService_ReturnsSelf() { - await using var innerSession = new TestRealtimeSession(); - await using var session = new OpenTelemetryRealtimeSession(innerSession); + await using var innerSession = new TestRealtimeClientSession(); + await using var session = new OpenTelemetryRealtimeClientSession(innerSession); - var self = session.GetService(typeof(OpenTelemetryRealtimeSession)); + var self = session.GetService(typeof(OpenTelemetryRealtimeClientSession)); Assert.Same(session, self); var realtime = session.GetService(typeof(IRealtimeClientSession)); @@ -363,7 +363,7 @@ public async Task TranscriptionSessionKind_Logged() .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { @@ -425,7 +425,7 @@ public async Task ToolChoiceMode_Logged(string modeKey, string expectedValue) .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { @@ -465,7 +465,7 @@ public async Task AIFunction_ForcedTool_Logged() .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { @@ -505,7 +505,7 @@ public async Task RequireAny_ToolMode_Logged() .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { @@ -545,7 +545,7 @@ public async Task NoToolChoice_NotLogged() .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { @@ -583,7 +583,7 @@ public async Task ToolCallContentInClientMessages_LoggedAsInputMessages() .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -626,7 +626,7 @@ public async Task ToolCallContentInServerMessages_LoggedAsOutputMessages() .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -669,7 +669,7 @@ public async Task ToolContentNotLoggedWithoutSensitiveData() .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -754,7 +754,7 @@ public async Task AudioBufferAppendMessage_LoggedAsInputMessage() .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -797,7 +797,7 @@ public async Task AudioBufferCommitMessage_LoggedAsInputMessage() .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -840,7 +840,7 @@ public async Task ResponseCreateMessageWithInstructions_LoggedAsInputMessage() .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -882,7 +882,7 @@ public async Task ResponseCreateMessageWithItems_LoggedAsInputMessage() .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -924,7 +924,7 @@ public async Task OutputTextAudioMessage_LoggedAsOutputMessage() .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -965,7 +965,7 @@ public async Task InputAudioTranscriptionMessage_LoggedAsOutputMessage() .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -1006,7 +1006,7 @@ public async Task ServerErrorMessage_LoggedAsOutputMessage() .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -1047,7 +1047,7 @@ public async Task ConversationItemCreateWithTextContent_LoggedAsInputMessage() .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -1088,7 +1088,7 @@ public async Task DataContentInClientMessage_LoggedWithModality() .AddInMemoryExporter(activities) .Build(); - await using var innerSession = new TestRealtimeSession + await using var innerSession = new TestRealtimeClientSession { Options = new RealtimeSessionOptions { Model = "test-model" }, GetServiceCallback = (serviceType, _) => @@ -1200,14 +1200,14 @@ private static async IAsyncEnumerable CallbackWithServerE public void UseOpenTelemetry_NullBuilder_Throws() { Assert.Throws("builder", () => - ((RealtimeSessionBuilder)null!).UseOpenTelemetry()); + ((RealtimeClientSessionBuilder)null!).UseOpenTelemetry()); } [Fact] public async Task UseOpenTelemetry_ConfigureCallback_IsInvoked() { - await using var innerSession = new TestRealtimeSession(); - var builder = new RealtimeSessionBuilder(innerSession); + await using var innerSession = new TestRealtimeClientSession(); + var builder = new RealtimeClientSessionBuilder(innerSession); bool configured = false; builder.UseOpenTelemetry(configure: session => @@ -1219,18 +1219,18 @@ public async Task UseOpenTelemetry_ConfigureCallback_IsInvoked() await using var pipeline = builder.Build(); Assert.True(configured); - var otelSession = pipeline.GetService(typeof(OpenTelemetryRealtimeSession)); + var otelSession = pipeline.GetService(typeof(OpenTelemetryRealtimeClientSession)); Assert.NotNull(otelSession); - var typedSession = Assert.IsType(otelSession); + var typedSession = Assert.IsType(otelSession); Assert.True(typedSession.EnableSensitiveData); } [Fact] public async Task DisposeAsync_CanBeCalledMultipleTimes() { - await using var innerSession = new TestRealtimeSession(); - var session = new OpenTelemetryRealtimeSession(innerSession); + await using var innerSession = new TestRealtimeClientSession(); + var session = new OpenTelemetryRealtimeClientSession(innerSession); await session.DisposeAsync(); await session.DisposeAsync(); diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientSessionBuilderTests.cs similarity index 72% rename from test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs rename to test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientSessionBuilderTests.cs index bfd3b7ee72a..979e53bd491 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionBuilderTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientSessionBuilderTests.cs @@ -12,25 +12,25 @@ namespace Microsoft.Extensions.AI; -public class RealtimeSessionBuilderTests +public class RealtimeClientSessionBuilderTests { [Fact] public void Ctor_NullSession_Throws() { - Assert.Throws("innerSession", () => new RealtimeSessionBuilder((IRealtimeClientSession)null!)); + Assert.Throws("innerSession", () => new RealtimeClientSessionBuilder((IRealtimeClientSession)null!)); } [Fact] public void Ctor_NullFactory_Throws() { - Assert.Throws("innerSessionFactory", () => new RealtimeSessionBuilder((Func)null!)); + Assert.Throws("innerSessionFactory", () => new RealtimeClientSessionBuilder((Func)null!)); } [Fact] public async Task Build_WithNoMiddleware_ReturnsInnerSession() { - await using var inner = new TestRealtimeSession(); - var builder = new RealtimeSessionBuilder(inner); + await using var inner = new TestRealtimeClientSession(); + var builder = new RealtimeClientSessionBuilder(inner); var result = builder.Build(); Assert.Same(inner, result); @@ -39,8 +39,8 @@ public async Task Build_WithNoMiddleware_ReturnsInnerSession() [Fact] public async Task Build_WithFactory_UsesFactory() { - await using var inner = new TestRealtimeSession(); - var builder = new RealtimeSessionBuilder(_ => inner); + await using var inner = new TestRealtimeClientSession(); + var builder = new RealtimeClientSessionBuilder(_ => inner); var result = builder.Build(); Assert.Same(inner, result); @@ -49,8 +49,8 @@ public async Task Build_WithFactory_UsesFactory() [Fact] public async Task Use_NullSessionFactory_Throws() { - await using var inner = new TestRealtimeSession(); - var builder = new RealtimeSessionBuilder(inner); + await using var inner = new TestRealtimeClientSession(); + var builder = new RealtimeClientSessionBuilder(inner); Assert.Throws("sessionFactory", () => builder.Use((Func)null!)); Assert.Throws("sessionFactory", () => builder.Use((Func)null!)); @@ -59,8 +59,8 @@ public async Task Use_NullSessionFactory_Throws() [Fact] public async Task Use_StreamingDelegate_NullFunc_Throws() { - await using var inner = new TestRealtimeSession(); - var builder = new RealtimeSessionBuilder(inner); + await using var inner = new TestRealtimeClientSession(); + var builder = new RealtimeClientSessionBuilder(inner); Assert.Throws( "getStreamingResponseFunc", @@ -71,19 +71,19 @@ public async Task Use_StreamingDelegate_NullFunc_Throws() public async Task Build_PipelineOrder_FirstAddedIsOutermost() { var callOrder = new List(); - await using var inner = new TestRealtimeSession(); + await using var inner = new TestRealtimeClientSession(); - var builder = new RealtimeSessionBuilder(inner); - builder.Use(session => new OrderTrackingSession(session, "first", callOrder)); - builder.Use(session => new OrderTrackingSession(session, "second", callOrder)); + var builder = new RealtimeClientSessionBuilder(inner); + builder.Use(session => new OrderTrackingClientSession(session, "first", callOrder)); + builder.Use(session => new OrderTrackingClientSession(session, "second", callOrder)); await using var pipeline = builder.Build(); // The outermost should be "first" (added first) - var outermost = Assert.IsType(pipeline); + var outermost = Assert.IsType(pipeline); Assert.Equal("first", outermost.Name); - var middle = Assert.IsType(outermost.GetInner()); + var middle = Assert.IsType(outermost.GetInner()); Assert.Equal("second", middle.Name); Assert.Same(inner, middle.GetInner()); @@ -93,9 +93,9 @@ public async Task Build_PipelineOrder_FirstAddedIsOutermost() public async Task Build_WithServiceProvider_PassesToFactory() { IServiceProvider? capturedServices = null; - await using var inner = new TestRealtimeSession(); + await using var inner = new TestRealtimeClientSession(); - var builder = new RealtimeSessionBuilder(inner); + var builder = new RealtimeClientSessionBuilder(inner); builder.Use((session, services) => { capturedServices = services; @@ -112,9 +112,9 @@ public async Task Build_WithServiceProvider_PassesToFactory() public async Task Build_NullServiceProvider_UsesEmptyProvider() { IServiceProvider? capturedServices = null; - await using var inner = new TestRealtimeSession(); + await using var inner = new TestRealtimeClientSession(); - var builder = new RealtimeSessionBuilder(inner); + var builder = new RealtimeClientSessionBuilder(inner); builder.Use((session, services) => { capturedServices = services; @@ -129,8 +129,8 @@ public async Task Build_NullServiceProvider_UsesEmptyProvider() [Fact] public async Task Use_ReturnsSameBuilder_ForChaining() { - await using var inner = new TestRealtimeSession(); - var builder = new RealtimeSessionBuilder(inner); + await using var inner = new TestRealtimeClientSession(); + var builder = new RealtimeClientSessionBuilder(inner); var returned = builder.Use(s => s); Assert.Same(builder, returned); @@ -140,12 +140,12 @@ public async Task Use_ReturnsSameBuilder_ForChaining() public async Task Use_WithStreamingDelegate_InterceptsStreaming() { var intercepted = false; - await using var inner = new TestRealtimeSession + await using var inner = new TestRealtimeClientSession { GetStreamingResponseAsyncCallback = (ct) => YieldSingle(new RealtimeServerMessage { MessageId = "inner" }, ct), }; - var builder = new RealtimeSessionBuilder(inner); + var builder = new RealtimeClientSessionBuilder(inner); builder.Use((innerSession, ct) => { intercepted = true; @@ -170,7 +170,7 @@ public void AsBuilder_NullSession_Throws() [Fact] public async Task AsBuilder_ReturnsBuilder() { - await using var inner = new TestRealtimeSession(); + await using var inner = new TestRealtimeClientSession(); var builder = inner.AsBuilder(); Assert.NotNull(builder); @@ -186,12 +186,12 @@ private static async IAsyncEnumerable YieldSingle( yield return message; } - private sealed class OrderTrackingSession : DelegatingRealtimeSession + private sealed class OrderTrackingClientSession : DelegatingRealtimeClientSession { public string Name { get; } private readonly List _callOrder; - public OrderTrackingSession(IRealtimeClientSession inner, string name, List callOrder) + public OrderTrackingClientSession(IRealtimeClientSession inner, string name, List callOrder) : base(inner) { Name = name; diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientSessionExtensionsTests.cs similarity index 71% rename from test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionExtensionsTests.cs rename to test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientSessionExtensionsTests.cs index 0e1986ee713..3970e8e8010 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeSessionExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientSessionExtensionsTests.cs @@ -9,7 +9,7 @@ namespace Microsoft.Extensions.AI; -public class RealtimeSessionExtensionsTests +public class RealtimeClientSessionExtensionsTests { [Fact] public void GetService_NullSession_Throws() @@ -20,15 +20,15 @@ public void GetService_NullSession_Throws() [Fact] public async Task GetService_ReturnsMatchingService() { - await using var session = new TestRealtimeSession(); - var result = session.GetService(); + await using var session = new TestRealtimeClientSession(); + var result = session.GetService(); Assert.Same(session, result); } [Fact] public async Task GetService_ReturnsNullForNonMatchingType() { - await using var session = new TestRealtimeSession(); + await using var session = new TestRealtimeClientSession(); var result = session.GetService(); Assert.Null(result); } @@ -36,15 +36,15 @@ public async Task GetService_ReturnsNullForNonMatchingType() [Fact] public async Task GetService_WithServiceKey_ReturnsNull() { - await using var session = new TestRealtimeSession(); - var result = session.GetService("someKey"); + await using var session = new TestRealtimeClientSession(); + var result = session.GetService("someKey"); Assert.Null(result); } [Fact] public async Task GetService_ReturnsInterfaceType() { - await using var session = new TestRealtimeSession(); + await using var session = new TestRealtimeClientSession(); var result = session.GetService(); Assert.Same(session, result); } From ddd058b59a7729357b29ccd4f39bafb20355be00 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Thu, 5 Mar 2026 14:11:09 -0800 Subject: [PATCH 61/92] Simplify DisposeAsyncCore to return ValueTask directly Remove unnecessary async/await in DelegatingRealtimeClientSession.DisposeAsyncCore, returning InnerSession.DisposeAsync() directly instead. --- .../Realtime/DelegatingRealtimeClientSession.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeClientSession.cs index 8018915dc7c..44415a77181 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeClientSession.cs @@ -41,11 +41,8 @@ public async ValueTask DisposeAsync() /// Performs async cleanup of managed resources. /// A task representing the asynchronous dispose operation. #pragma warning disable EA0014 // The async method doesn't support cancellation - protected virtual async ValueTask DisposeAsyncCore() + protected virtual ValueTask DisposeAsyncCore() => InnerSession.DisposeAsync(); #pragma warning restore EA0014 - { - await InnerSession.DisposeAsync().ConfigureAwait(false); - } /// Gets the inner . protected IRealtimeClientSession InnerSession { get; } From 0161bf715000eacfece97d409d810923e89260f9 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Thu, 5 Mar 2026 14:51:34 -0800 Subject: [PATCH 62/92] Convert RealtimeSessionKind from enum to extensible string struct Convert RealtimeSessionKind from a closed enum to a readonly struct following the same pattern as RealtimeServerMessageType and ChatFinishReason. This allows providers to define custom session kinds. Rename Realtime -> Conversation for clarity, avoiding the redundant RealtimeSessionKind.Realtime naming. --- .../Realtime/RealtimeSessionKind.cs | 87 +++++++++++++++++-- .../Realtime/RealtimeSessionOptions.cs | 2 +- .../OpenAIRealtimeClientSession.cs | 2 +- .../Realtime/RealtimeSessionOptionsTests.cs | 2 +- ...OpenTelemetryRealtimeClientSessionTests.cs | 6 +- 5 files changed, 87 insertions(+), 12 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionKind.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionKind.cs index 120c7787a81..c612ac08c5a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionKind.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionKind.cs @@ -1,24 +1,99 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; +using System.ComponentModel; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; /// -/// Represents options for configuring a real-time session. +/// Represents the kind of a real-time session. /// +/// +/// Well-known session kinds are provided as static properties. Providers may define additional +/// session kinds by constructing new instances with custom values. +/// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public enum RealtimeSessionKind +[JsonConverter(typeof(Converter))] +[DebuggerDisplay("{Value,nq}")] +public readonly struct RealtimeSessionKind : IEquatable { /// - /// Represent a realtime sessions which process audio, text, or other media in real-time. + /// Gets a session kind representing a conversational session which processes audio, text, or other media in real-time. /// - Realtime, + public static RealtimeSessionKind Conversation { get; } = new("conversation"); /// - /// Represent transcription only session. + /// Gets a session kind representing a transcription-only session. /// - Transcription + public static RealtimeSessionKind Transcription { get; } = new("transcription"); + + /// Gets the value of the session kind. + public string Value { get; } + + /// Initializes a new instance of the struct with the provided value. + /// The value to associate with this . + [JsonConstructor] + public RealtimeSessionKind(string value) + { + Value = Throw.IfNullOrWhitespace(value); + } + + /// + /// Returns a value indicating whether two instances are equivalent, as determined by a + /// case-insensitive comparison of their values. + /// + /// The first instance to compare. + /// The second instance to compare. + /// if left and right have equivalent values; otherwise, . + public static bool operator ==(RealtimeSessionKind left, RealtimeSessionKind right) + { + return left.Equals(right); + } + + /// + /// Returns a value indicating whether two instances are not equivalent, as determined by a + /// case-insensitive comparison of their values. + /// + /// The first instance to compare. + /// The second instance to compare. + /// if left and right have different values; otherwise, . + public static bool operator !=(RealtimeSessionKind left, RealtimeSessionKind right) + { + return !(left == right); + } + + /// + public override bool Equals([NotNullWhen(true)] object? obj) + => obj is RealtimeSessionKind other && Equals(other); + + /// + public bool Equals(RealtimeSessionKind other) + => string.Equals(Value, other.Value, StringComparison.OrdinalIgnoreCase); + + /// + public override int GetHashCode() + => Value is null ? 0 : StringComparer.OrdinalIgnoreCase.GetHashCode(Value); + + /// + public override string ToString() => Value ?? string.Empty; + + /// Provides a for serializing instances. + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class Converter : JsonConverter + { + /// + public override RealtimeSessionKind Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + new(reader.GetString()!); + + /// + public override void Write(Utf8JsonWriter writer, RealtimeSessionKind value, JsonSerializerOptions options) => + Throw.IfNull(writer).WriteStringValue(value.Value); + } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs index 65c77dc1ef3..21a1e952911 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs @@ -19,7 +19,7 @@ public class RealtimeSessionOptions /// /// If set to , most of the sessions properties will not apply to the session. Only InputAudioFormat, NoiseReductionOptions, TranscriptionOptions, and VoiceActivityDetection will be used. /// - public RealtimeSessionKind SessionKind { get; init; } = RealtimeSessionKind.Realtime; + public RealtimeSessionKind SessionKind { get; init; } = RealtimeSessionKind.Conversation; /// /// Gets the model name to use for the session. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs index f0655b99743..d96e198c3eb 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs @@ -990,7 +990,7 @@ private RealtimeSessionOptions MapConversationSessionToOptions(Sdk.RealtimeConve return new RealtimeSessionOptions { - SessionKind = RealtimeSessionKind.Realtime, + SessionKind = RealtimeSessionKind.Conversation, Model = session.Model, Instructions = session.Instructions, MaxOutputTokens = maxOutputTokens, diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs index ae2f005e454..79c1f92b35c 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs @@ -15,7 +15,7 @@ public void Constructor_Parameterless_PropsDefaulted() { RealtimeSessionOptions options = new(); - Assert.Equal(RealtimeSessionKind.Realtime, options.SessionKind); + Assert.Equal(RealtimeSessionKind.Conversation, options.SessionKind); Assert.Null(options.Model); Assert.Null(options.InputAudioFormat); Assert.Null(options.NoiseReductionOptions); diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientSessionTests.cs index 0c68e37dd50..a9d1fe76f6b 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientSessionTests.cs @@ -37,7 +37,7 @@ public async Task ExpectedInformationLogged_GetStreamingResponseAsync(bool enabl MaxOutputTokens = 500, OutputModalities = ["text", "audio"], Instructions = "Be helpful and friendly.", - SessionKind = RealtimeSessionKind.Realtime, + SessionKind = RealtimeSessionKind.Conversation, Tools = [AIFunctionFactory.Create((string query) => query, "Search", "Search for information.")], }, GetServiceCallback = (serviceType, serviceKey) => @@ -123,7 +123,7 @@ static async IAsyncEnumerable CallbackAsync([EnumeratorCa Assert.Equal(500, activity.GetTagItem("gen_ai.request.max_tokens")); // Realtime-specific attributes - Assert.Equal("Realtime", activity.GetTagItem("gen_ai.realtime.session_kind")); + Assert.Equal("conversation", activity.GetTagItem("gen_ai.realtime.session_kind")); Assert.Equal("alloy", activity.GetTagItem("gen_ai.realtime.voice")); Assert.Equal("""["text", "audio"]""", activity.GetTagItem("gen_ai.realtime.output_modalities")); @@ -401,7 +401,7 @@ static async IAsyncEnumerable TranscriptionCallbackAsync( } var activity = Assert.Single(activities); - Assert.Equal("Transcription", activity.GetTagItem("gen_ai.realtime.session_kind")); + Assert.Equal("transcription", activity.GetTagItem("gen_ai.realtime.session_kind")); } [Theory] From d16fa7ca6aaf4991ec92c40498e37dee7f8d88ef Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Thu, 5 Mar 2026 15:07:31 -0800 Subject: [PATCH 63/92] Remove NoiseReductionOptions from abstraction NoiseReductionOptions is OpenAI-specific and not supported by other providers. Remove it from the abstraction layer; users who need noise reduction can use provider-specific options. --- .../Realtime/NoiseReductionOptions.cs | 29 ------------------- .../Realtime/RealtimeSessionOptions.cs | 7 +---- .../OpenAIRealtimeClientSession.cs | 27 +---------------- .../Realtime/RealtimeSessionOptionsTests.cs | 3 -- 4 files changed, 2 insertions(+), 64 deletions(-) delete mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/NoiseReductionOptions.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/NoiseReductionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/NoiseReductionOptions.cs deleted file mode 100644 index d39d759f462..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/NoiseReductionOptions.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics.CodeAnalysis; -using Microsoft.Shared.DiagnosticIds; - -namespace Microsoft.Extensions.AI; - -/// -/// Represents options for configuring a real-time session. -/// -[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public enum NoiseReductionOptions -{ - /// - /// No noise reduction applied. - /// - None, - - /// - /// for close-talking microphones. - /// - NearField, - - /// - /// For far-field microphones. - /// - FarField -} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs index 21a1e952911..78971d61aa9 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs @@ -17,7 +17,7 @@ public class RealtimeSessionOptions /// Gets the session kind. /// /// - /// If set to , most of the sessions properties will not apply to the session. Only InputAudioFormat, NoiseReductionOptions, TranscriptionOptions, and VoiceActivityDetection will be used. + /// If set to , most of the sessions properties will not apply to the session. Only InputAudioFormat, TranscriptionOptions, and VoiceActivityDetection will be used. /// public RealtimeSessionKind SessionKind { get; init; } = RealtimeSessionKind.Conversation; @@ -31,11 +31,6 @@ public class RealtimeSessionOptions /// public RealtimeAudioFormat? InputAudioFormat { get; init; } - /// - /// Gets the noise reduction options for the session. - /// - public NoiseReductionOptions? NoiseReductionOptions { get; init; } - /// /// Gets the transcription options for the session. /// diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs index d96e198c3eb..fbe1c685f9f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs @@ -394,14 +394,6 @@ private static Sdk.RealtimeConversationSessionOptions BuildConversationSessionOp inputAudioOptions.AudioFormat = ToSdkAudioFormat(options.InputAudioFormat); } - if (options.NoiseReductionOptions.HasValue) - { - inputAudioOptions.NoiseReduction = new Sdk.RealtimeNoiseReduction( - options.NoiseReductionOptions.Value == NoiseReductionOptions.NearField - ? Sdk.RealtimeNoiseReductionKind.NearField - : Sdk.RealtimeNoiseReductionKind.FarField); - } - if (options.TranscriptionOptions is not null) { inputAudioOptions.AudioTranscriptionOptions = new Sdk.RealtimeAudioTranscriptionOptions @@ -510,7 +502,7 @@ private static Sdk.RealtimeTranscriptionSessionOptions BuildTranscriptionSession var transOptions = new Sdk.RealtimeTranscriptionSessionOptions(); if (options.InputAudioFormat is not null || options.TranscriptionOptions is not null || - options.VoiceActivityDetection is not null || options.NoiseReductionOptions.HasValue) + options.VoiceActivityDetection is not null) { var inputAudioOptions = new Sdk.RealtimeTranscriptionSessionInputAudioOptions(); @@ -529,14 +521,6 @@ private static Sdk.RealtimeTranscriptionSessionOptions BuildTranscriptionSession }; } - if (options.NoiseReductionOptions.HasValue) - { - inputAudioOptions.NoiseReduction = new Sdk.RealtimeNoiseReduction( - options.NoiseReductionOptions.Value == NoiseReductionOptions.NearField - ? Sdk.RealtimeNoiseReductionKind.NearField - : Sdk.RealtimeNoiseReductionKind.FarField); - } - if (options.VoiceActivityDetection is ServerVoiceActivityDetection serverVad) { inputAudioOptions.TurnDetection = new Sdk.RealtimeServerVadTurnDetection @@ -904,7 +888,6 @@ private RealtimeServerMessage HandleSessionEvent(Sdk.RealtimeSession? session, S private RealtimeSessionOptions MapConversationSessionToOptions(Sdk.RealtimeConversationSession session) { RealtimeAudioFormat? inputAudioFormat = null; - NoiseReductionOptions? noiseReduction = null; TranscriptionOptions? transcription = null; VoiceActivityDetection? vad = null; RealtimeAudioFormat? outputAudioFormat = null; @@ -916,13 +899,6 @@ private RealtimeSessionOptions MapConversationSessionToOptions(Sdk.RealtimeConve { inputAudioFormat = MapSdkAudioFormat(inputOpts.AudioFormat); - if (inputOpts.NoiseReduction is { } nr) - { - noiseReduction = nr.Kind == Sdk.RealtimeNoiseReductionKind.NearField - ? NoiseReductionOptions.NearField - : NoiseReductionOptions.FarField; - } - if (inputOpts.AudioTranscriptionOptions is { } transcriptionOpts) { transcription = new TranscriptionOptions @@ -996,7 +972,6 @@ private RealtimeSessionOptions MapConversationSessionToOptions(Sdk.RealtimeConve MaxOutputTokens = maxOutputTokens, OutputModalities = outputModalities, InputAudioFormat = inputAudioFormat, - NoiseReductionOptions = noiseReduction, TranscriptionOptions = transcription, VoiceActivityDetection = vad, OutputAudioFormat = outputAudioFormat, diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs index 79c1f92b35c..53e7cc1ba1f 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs @@ -18,7 +18,6 @@ public void Constructor_Parameterless_PropsDefaulted() Assert.Equal(RealtimeSessionKind.Conversation, options.SessionKind); Assert.Null(options.Model); Assert.Null(options.InputAudioFormat); - Assert.Null(options.NoiseReductionOptions); Assert.Null(options.TranscriptionOptions); Assert.Null(options.VoiceActivityDetection); Assert.Null(options.OutputAudioFormat); @@ -46,7 +45,6 @@ public void Properties_Roundtrip() Model = "gpt-4-realtime", InputAudioFormat = inputFormat, OutputAudioFormat = outputFormat, - NoiseReductionOptions = NoiseReductionOptions.NearField, TranscriptionOptions = transcriptionOptions, VoiceActivityDetection = vad, Voice = "alloy", @@ -61,7 +59,6 @@ public void Properties_Roundtrip() Assert.Equal("gpt-4-realtime", options.Model); Assert.Same(inputFormat, options.InputAudioFormat); Assert.Same(outputFormat, options.OutputAudioFormat); - Assert.Equal(NoiseReductionOptions.NearField, options.NoiseReductionOptions); Assert.Same(transcriptionOptions, options.TranscriptionOptions); Assert.Same(vad, options.VoiceActivityDetection); Assert.Equal("alloy", options.Voice); From a2dd558f057bcb9de73c6fc12ae7b2fa61c43437 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Thu, 5 Mar 2026 15:21:22 -0800 Subject: [PATCH 64/92] Remove VoiceActivityDetection hierarchy from abstraction VoiceActivityDetection, ServerVoiceActivityDetection, and SemanticVoiceActivityDetection are too OpenAI-specific. Remove them from the abstraction; users who need VAD can configure it through provider-specific options via RawRepresentationFactory. --- .../Realtime/RealtimeSessionOptions.cs | 7 +- .../SemanticVoiceActivityDetection.cs | 15 --- .../Realtime/ServerVoiceActivityDetection.cs | 37 ------ .../Realtime/VoiceActivityDetection.cs | 27 ----- .../OpenAIRealtimeClientSession.cs | 109 +----------------- .../Realtime/RealtimeSessionOptionsTests.cs | 17 --- 6 files changed, 2 insertions(+), 210 deletions(-) delete mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs delete mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs delete mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs index 78971d61aa9..4f1ad9c9242 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs @@ -17,7 +17,7 @@ public class RealtimeSessionOptions /// Gets the session kind. /// /// - /// If set to , most of the sessions properties will not apply to the session. Only InputAudioFormat, TranscriptionOptions, and VoiceActivityDetection will be used. + /// If set to , most of the sessions properties will not apply to the session. Only InputAudioFormat and TranscriptionOptions will be used. /// public RealtimeSessionKind SessionKind { get; init; } = RealtimeSessionKind.Conversation; @@ -36,11 +36,6 @@ public class RealtimeSessionOptions /// public TranscriptionOptions? TranscriptionOptions { get; init; } - /// - /// Gets the voice activity detection options for the session. - /// - public VoiceActivityDetection? VoiceActivityDetection { get; init; } - /// /// Gets the output audio format for the session. /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs deleted file mode 100644 index 5f2d3678def..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SemanticVoiceActivityDetection.cs +++ /dev/null @@ -1,15 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics.CodeAnalysis; -using Microsoft.Shared.DiagnosticIds; - -namespace Microsoft.Extensions.AI; - -/// -/// Represents options for configuring server voice activity detection in a real-time session. -/// -[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class SemanticVoiceActivityDetection : VoiceActivityDetection -{ -} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs deleted file mode 100644 index 7b0946337ef..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ServerVoiceActivityDetection.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics.CodeAnalysis; -using Microsoft.Shared.DiagnosticIds; - -namespace Microsoft.Extensions.AI; - -/// -/// Represents options for configuring server voice activity detection in a real-time session. -/// -[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class ServerVoiceActivityDetection : VoiceActivityDetection -{ - /// - /// Gets the idle timeout in milliseconds to detect the end of speech. - /// - public int IdleTimeoutInMilliseconds { get; init; } - - /// - /// Gets the prefix padding in milliseconds to include before detected speech. - /// - public int PrefixPaddingInMilliseconds { get; init; } = 300; - - /// - /// Gets the silence duration in milliseconds to consider as a pause. - /// - public int SilenceDurationInMilliseconds { get; init; } = 500; - - /// - /// Gets the threshold for voice activity detection. - /// - /// - /// A value between 0.0 and 1.0, where higher values make the detection more sensitive. - /// - public double Threshold { get; init; } = 0.5; -} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs deleted file mode 100644 index c0958f68579..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetection.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics.CodeAnalysis; -using Microsoft.Shared.DiagnosticIds; - -namespace Microsoft.Extensions.AI; - -/// -/// Represents options for configuring voice activity detection in a real-time session. -/// -[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class VoiceActivityDetection -{ - /// - /// Gets a value indicating whether to create a response when voice activity is detected. - /// - public bool CreateResponse { get; init; } - - /// - /// Gets a value indicating whether to interrupt the response when voice activity is detected. - /// - public bool InterruptResponse { get; init; } - - /// Gets or sets any additional properties associated with the voice activity detection options. - public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } -} diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs index fbe1c685f9f..f55be445d1f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs @@ -404,43 +404,6 @@ private static Sdk.RealtimeConversationSessionOptions BuildConversationSessionOp }; } - if (options.VoiceActivityDetection is ServerVoiceActivityDetection serverVad) - { - inputAudioOptions.TurnDetection = new Sdk.RealtimeServerVadTurnDetection - { - CreateResponseEnabled = serverVad.CreateResponse, - InterruptResponseEnabled = serverVad.InterruptResponse, - DetectionThreshold = (float)serverVad.Threshold, - IdleTimeout = TimeSpan.FromMilliseconds(serverVad.IdleTimeoutInMilliseconds), - PrefixPadding = TimeSpan.FromMilliseconds(serverVad.PrefixPaddingInMilliseconds), - SilenceDuration = TimeSpan.FromMilliseconds(serverVad.SilenceDurationInMilliseconds), - }; - } - else if (options.VoiceActivityDetection is SemanticVoiceActivityDetection semanticVad) - { - var turnDetection = new Sdk.RealtimeSemanticVadTurnDetection - { - CreateResponseEnabled = semanticVad.CreateResponse, - InterruptResponseEnabled = semanticVad.InterruptResponse, - }; - - if (semanticVad.AdditionalProperties?.TryGetValue("eagerness", out var eagerness) is true && eagerness is string eagernessStr) - { - turnDetection.EagernessLevel = new Sdk.RealtimeSemanticVadEagernessLevel(eagernessStr); - } - - inputAudioOptions.TurnDetection = turnDetection; - } - else if (options.VoiceActivityDetection is { } baseVad) - { - // Base VoiceActivityDetection: default to server VAD with basic settings. - inputAudioOptions.TurnDetection = new Sdk.RealtimeServerVadTurnDetection - { - CreateResponseEnabled = baseVad.CreateResponse, - InterruptResponseEnabled = baseVad.InterruptResponse, - }; - } - if (options.OutputAudioFormat is not null) { outputAudioOptions.AudioFormat = ToSdkAudioFormat(options.OutputAudioFormat); @@ -501,8 +464,7 @@ private static Sdk.RealtimeTranscriptionSessionOptions BuildTranscriptionSession { var transOptions = new Sdk.RealtimeTranscriptionSessionOptions(); - if (options.InputAudioFormat is not null || options.TranscriptionOptions is not null || - options.VoiceActivityDetection is not null) + if (options.InputAudioFormat is not null || options.TranscriptionOptions is not null) { var inputAudioOptions = new Sdk.RealtimeTranscriptionSessionInputAudioOptions(); @@ -521,42 +483,6 @@ private static Sdk.RealtimeTranscriptionSessionOptions BuildTranscriptionSession }; } - if (options.VoiceActivityDetection is ServerVoiceActivityDetection serverVad) - { - inputAudioOptions.TurnDetection = new Sdk.RealtimeServerVadTurnDetection - { - CreateResponseEnabled = serverVad.CreateResponse, - InterruptResponseEnabled = serverVad.InterruptResponse, - DetectionThreshold = (float)serverVad.Threshold, - IdleTimeout = TimeSpan.FromMilliseconds(serverVad.IdleTimeoutInMilliseconds), - PrefixPadding = TimeSpan.FromMilliseconds(serverVad.PrefixPaddingInMilliseconds), - SilenceDuration = TimeSpan.FromMilliseconds(serverVad.SilenceDurationInMilliseconds), - }; - } - else if (options.VoiceActivityDetection is SemanticVoiceActivityDetection semanticVad) - { - var turnDetection = new Sdk.RealtimeSemanticVadTurnDetection - { - CreateResponseEnabled = semanticVad.CreateResponse, - InterruptResponseEnabled = semanticVad.InterruptResponse, - }; - - if (semanticVad.AdditionalProperties?.TryGetValue("eagerness", out var eagerness) is true && eagerness is string eagernessStr) - { - turnDetection.EagernessLevel = new Sdk.RealtimeSemanticVadEagernessLevel(eagernessStr); - } - - inputAudioOptions.TurnDetection = turnDetection; - } - else if (options.VoiceActivityDetection is { } baseVad) - { - inputAudioOptions.TurnDetection = new Sdk.RealtimeServerVadTurnDetection - { - CreateResponseEnabled = baseVad.CreateResponse, - InterruptResponseEnabled = baseVad.InterruptResponse, - }; - } - transOptions.AudioOptions = new Sdk.RealtimeTranscriptionSessionAudioOptions { InputAudioOptions = inputAudioOptions, @@ -889,7 +815,6 @@ private RealtimeSessionOptions MapConversationSessionToOptions(Sdk.RealtimeConve { RealtimeAudioFormat? inputAudioFormat = null; TranscriptionOptions? transcription = null; - VoiceActivityDetection? vad = null; RealtimeAudioFormat? outputAudioFormat = null; string? voice = null; @@ -908,37 +833,6 @@ private RealtimeSessionOptions MapConversationSessionToOptions(Sdk.RealtimeConve Prompt = transcriptionOpts.Prompt, }; } - - if (inputOpts.TurnDetection is Sdk.RealtimeServerVadTurnDetection serverVad) - { - vad = new ServerVoiceActivityDetection - { - CreateResponse = serverVad.CreateResponseEnabled ?? false, - InterruptResponse = serverVad.InterruptResponseEnabled ?? false, - Threshold = serverVad.DetectionThreshold ?? 0.5, - IdleTimeoutInMilliseconds = (int)(serverVad.IdleTimeout?.TotalMilliseconds ?? 0), - PrefixPaddingInMilliseconds = (int)(serverVad.PrefixPadding?.TotalMilliseconds ?? 300), - SilenceDurationInMilliseconds = (int)(serverVad.SilenceDuration?.TotalMilliseconds ?? 500), - }; - } - else if (inputOpts.TurnDetection is Sdk.RealtimeSemanticVadTurnDetection semanticVad) - { - var semanticVadOptions = new SemanticVoiceActivityDetection - { - CreateResponse = semanticVad.CreateResponseEnabled ?? false, - InterruptResponse = semanticVad.InterruptResponseEnabled ?? false, - }; - - if (semanticVad.EagernessLevel.HasValue) - { - semanticVadOptions.AdditionalProperties = new AdditionalPropertiesDictionary - { - ["eagerness"] = semanticVad.EagernessLevel.Value.ToString(), - }; - } - - vad = semanticVadOptions; - } } if (audioOptions.OutputAudioOptions is { } outputOpts) @@ -973,7 +867,6 @@ private RealtimeSessionOptions MapConversationSessionToOptions(Sdk.RealtimeConve OutputModalities = outputModalities, InputAudioFormat = inputAudioFormat, TranscriptionOptions = transcription, - VoiceActivityDetection = vad, OutputAudioFormat = outputAudioFormat, Voice = voice, diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs index 53e7cc1ba1f..059bbc4c7d2 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs @@ -19,7 +19,6 @@ public void Constructor_Parameterless_PropsDefaulted() Assert.Null(options.Model); Assert.Null(options.InputAudioFormat); Assert.Null(options.TranscriptionOptions); - Assert.Null(options.VoiceActivityDetection); Assert.Null(options.OutputAudioFormat); Assert.Null(options.Voice); Assert.Null(options.Instructions); @@ -37,7 +36,6 @@ public void Properties_Roundtrip() List modalities = ["text", "audio"]; List tools = [AIFunctionFactory.Create(() => 42)]; var transcriptionOptions = new TranscriptionOptions { SpeechLanguage = "en", ModelId = "whisper-1", Prompt = "greeting" }; - var vad = new VoiceActivityDetection { CreateResponse = true, InterruptResponse = true }; RealtimeSessionOptions options = new() { @@ -46,7 +44,6 @@ public void Properties_Roundtrip() InputAudioFormat = inputFormat, OutputAudioFormat = outputFormat, TranscriptionOptions = transcriptionOptions, - VoiceActivityDetection = vad, Voice = "alloy", Instructions = "Be helpful", MaxOutputTokens = 500, @@ -60,7 +57,6 @@ public void Properties_Roundtrip() Assert.Same(inputFormat, options.InputAudioFormat); Assert.Same(outputFormat, options.OutputAudioFormat); Assert.Same(transcriptionOptions, options.TranscriptionOptions); - Assert.Same(vad, options.VoiceActivityDetection); Assert.Equal("alloy", options.Voice); Assert.Equal("Be helpful", options.Instructions); Assert.Equal(500, options.MaxOutputTokens); @@ -94,17 +90,4 @@ public void TranscriptionOptions_PromptDefaultsToNull() Assert.Null(options.Prompt); } - [Fact] - public void VoiceActivityDetection_Properties_Roundtrip() - { - var vad = new VoiceActivityDetection(); - - Assert.False(vad.CreateResponse); - Assert.False(vad.InterruptResponse); - - var vad2 = new VoiceActivityDetection { CreateResponse = true, InterruptResponse = true }; - - Assert.True(vad2.CreateResponse); - Assert.True(vad2.InterruptResponse); - } } From 0f160d2f36967546ee96bc063876fa0265ed2e24 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Thu, 5 Mar 2026 16:20:27 -0800 Subject: [PATCH 65/92] Remove AnonymousDelegatingRealtimeClientSession and anonymous Use overload Remove the anonymous Use overload that only intercepts GetStreamingResponseAsync without the ability to intercept SendAsync, making it too limited to be useful in practice. --- ...nonymousDelegatingRealtimeClientSession.cs | 42 ----------------- .../Realtime/RealtimeClientSessionBuilder.cs | 24 ---------- .../RealtimeClientSessionBuilderTests.cs | 45 ------------------- 3 files changed, 111 deletions(-) delete mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeClientSession.cs diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeClientSession.cs deleted file mode 100644 index 8ee05cd05aa..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/AnonymousDelegatingRealtimeClientSession.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Threading; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Extensions.AI; - -/// Represents a delegating realtime session that wraps an inner session with implementations provided by delegates. -[Experimental("MEAI001")] -internal sealed class AnonymousDelegatingRealtimeClientSession : DelegatingRealtimeClientSession -{ - /// The delegate to use as the implementation of . - private readonly Func> _getStreamingResponseFunc; - - /// - /// Initializes a new instance of the class. - /// - /// The inner session. - /// - /// A delegate that provides the implementation for . - /// - /// is . - /// is . - public AnonymousDelegatingRealtimeClientSession( - IRealtimeClientSession innerSession, - Func> getStreamingResponseFunc) - : base(innerSession) - { - _getStreamingResponseFunc = Throw.IfNull(getStreamingResponseFunc); - } - - /// - public override IAsyncEnumerable GetStreamingResponseAsync( - CancellationToken cancellationToken = default) - { - return _getStreamingResponseFunc(InnerSession, cancellationToken); - } -} diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionBuilder.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionBuilder.cs index 2711e09c0bf..f5ed69f2a21 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionBuilder.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionBuilder.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -using System.Threading; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; @@ -86,27 +85,4 @@ public RealtimeClientSessionBuilder Use(Func - /// Adds to the realtime session pipeline an anonymous delegating realtime session based on a delegate that provides - /// an implementation for . - /// - /// - /// A delegate that provides the implementation for . - /// This delegate is invoked with a delegate that represents invoking - /// the inner session, and a cancellation token. The delegate should be passed whatever - /// cancellation token should be passed along to the next stage in the pipeline. - /// - /// The updated instance. - /// - /// This overload can be used when the anonymous implementation needs to provide pre-processing and/or post-processing - /// for the streaming response. - /// - /// is . - public RealtimeClientSessionBuilder Use( - Func> getStreamingResponseFunc) - { - _ = Throw.IfNull(getStreamingResponseFunc); - - return Use((innerSession, _) => new AnonymousDelegatingRealtimeClientSession(innerSession, getStreamingResponseFunc)); - } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientSessionBuilderTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientSessionBuilderTests.cs index 979e53bd491..0ba2ae72d32 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientSessionBuilderTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientSessionBuilderTests.cs @@ -56,17 +56,6 @@ public async Task Use_NullSessionFactory_Throws() Assert.Throws("sessionFactory", () => builder.Use((Func)null!)); } - [Fact] - public async Task Use_StreamingDelegate_NullFunc_Throws() - { - await using var inner = new TestRealtimeClientSession(); - var builder = new RealtimeClientSessionBuilder(inner); - - Assert.Throws( - "getStreamingResponseFunc", - () => builder.Use((Func>)null!)); - } - [Fact] public async Task Build_PipelineOrder_FirstAddedIsOutermost() { @@ -136,31 +125,6 @@ public async Task Use_ReturnsSameBuilder_ForChaining() Assert.Same(builder, returned); } - [Fact] - public async Task Use_WithStreamingDelegate_InterceptsStreaming() - { - var intercepted = false; - await using var inner = new TestRealtimeClientSession - { - GetStreamingResponseAsyncCallback = (ct) => YieldSingle(new RealtimeServerMessage { MessageId = "inner" }, ct), - }; - - var builder = new RealtimeClientSessionBuilder(inner); - builder.Use((innerSession, ct) => - { - intercepted = true; - return innerSession.GetStreamingResponseAsync(ct); - }); - - await using var pipeline = builder.Build(); - await foreach (var msg in pipeline.GetStreamingResponseAsync()) - { - Assert.Equal("inner", msg.MessageId); - } - - Assert.True(intercepted); - } - [Fact] public void AsBuilder_NullSession_Throws() { @@ -177,15 +141,6 @@ public async Task AsBuilder_ReturnsBuilder() Assert.Same(inner, builder.Build()); } - private static async IAsyncEnumerable YieldSingle( - RealtimeServerMessage message, - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - _ = cancellationToken; - await Task.CompletedTask.ConfigureAwait(false); - yield return message; - } - private sealed class OrderTrackingClientSession : DelegatingRealtimeClientSession { public string Name { get; } From 2f4a0b209f3e51709ac77e88b46388cdd7e7c986 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Thu, 5 Mar 2026 16:47:23 -0800 Subject: [PATCH 66/92] Fix JSON injection in MessageId by using JsonSerializer.Serialize --- .../OpenAIRealtimeClientSession.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs index f55be445d1f..1feff1e2316 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs @@ -373,7 +373,7 @@ private async Task SendRawCommandAsync(RealtimeClientMessage message, Cancellati // Inject event_id if the message has one but the raw JSON does not. if (message.MessageId is not null && !jsonString.Contains("\"event_id\"", StringComparison.Ordinal)) { - jsonString = jsonString.Insert(1, $"\"event_id\":\"{message.MessageId}\","); + jsonString = jsonString.Insert(1, $"\"event_id\":{JsonSerializer.Serialize(message.MessageId)},"); } await _sessionClient!.SendCommandAsync(BinaryData.FromString(jsonString), null).ConfigureAwait(false); From 50a773c9470611357d93c871b73be035be9333e9 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Thu, 5 Mar 2026 17:12:06 -0800 Subject: [PATCH 67/92] Remove meaningless duration metric from SendAsync in OTel session The stopwatch was started and recorded within the same block without wrapping the actual SendAsync call, measuring nothing useful. Duration metrics remain in GetStreamingResponseAsync where they measure actual streaming time. --- .../Realtime/OpenTelemetryRealtimeClientSession.cs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs index 1b54bd76562..875ca61ba18 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs @@ -178,23 +178,11 @@ public override async Task SendAsync(RealtimeClientMessage message, Cancellation if (otelMessage is not null) { - RealtimeSessionOptions? options = Options; - string? requestModelId = options?.Model ?? _defaultModelId; - Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; - using Activity? inputActivity = CreateAndConfigureActivity(options: null); if (inputActivity is { IsAllDataRequested: true }) { _ = inputActivity.AddTag(OpenTelemetryConsts.GenAI.Input.Messages, SerializeMessage(otelMessage)); } - - // Record metrics - if (_operationDurationHistogram.Enabled && stopwatch is not null) - { - TagList tags = default; - AddMetricTags(ref tags, requestModelId, responseModelId: null); - _operationDurationHistogram.Record(stopwatch.Elapsed.TotalSeconds, tags); - } } } From 59321107e4d9bcaa56f2aa7b4aead13d9cf94ca7 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Fri, 6 Mar 2026 09:36:49 -0800 Subject: [PATCH 68/92] Refactor builder to operate on IRealtimeClient instead of IRealtimeClientSession - Create DelegatingRealtimeClient base class (replaces DelegatingRealtimeClientSession) - Create RealtimeClientBuilder (replaces RealtimeClientSessionBuilder) - Create FunctionInvokingRealtimeClient, LoggingRealtimeClient, OpenTelemetryRealtimeClient - Make session middleware types (FunctionInvokingRealtimeClientSession, etc.) internal - Rename builder extension classes to remove 'Session' from names - Add RealtimeClientExtensions and RealtimeClientBuilderRealtimeClientExtensions - Use DiagnosticIds.Experiments.AIRealTime for Experimental attributes - Update all tests to use public client APIs only --- .../Realtime/DelegatingRealtimeClient.cs | 68 +++++ .../DelegatingRealtimeClientSession.cs | 72 ------ .../FunctionInvokingRealtimeClient.cs | 131 ++++++++++ ...nvokingRealtimeClientBuilderExtensions.cs} | 28 +- .../FunctionInvokingRealtimeClientSession.cs | 207 ++++----------- .../Realtime/LoggingRealtimeClient.cs | 56 ++++ ...LoggingRealtimeClientBuilderExtensions.cs} | 34 +-- .../Realtime/LoggingRealtimeClientSession.cs | 36 ++- .../Realtime/OpenTelemetryRealtimeClient.cs | 71 ++++++ ...lemetryRealtimeClientBuilderExtensions.cs} | 31 +-- .../OpenTelemetryRealtimeClientSession.cs | 40 +-- .../Realtime/RealtimeClientBuilder.cs | 89 +++++++ ...meClientBuilderRealtimeClientExtensions.cs | 29 +++ .../Realtime/RealtimeClientExtensions.cs | 31 +++ .../Realtime/RealtimeClientSessionBuilder.cs | 88 ------- ...nBuilderRealtimeClientSessionExtensions.cs | 28 -- .../RealtimeClientSessionExtensions.cs | 3 +- .../DelegatingRealtimeClientSessionTests.cs | 240 ------------------ ...=> FunctionInvokingRealtimeClientTests.cs} | 182 +++++++------ ...Tests.cs => LoggingRealtimeClientTests.cs} | 123 ++++++--- ...cs => OpenTelemetryRealtimeClientTests.cs} | 214 +++++++++------- .../Realtime/RealtimeClientBuilderTests.cs | 182 +++++++++++++ .../RealtimeClientSessionBuilderTests.cs | 173 ------------- 23 files changed, 1116 insertions(+), 1040 deletions(-) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeClient.cs delete mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeClientSession.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClient.cs rename src/Libraries/Microsoft.Extensions.AI/Realtime/{FunctionInvokingRealtimeClientSessionBuilderExtensions.cs => FunctionInvokingRealtimeClientBuilderExtensions.cs} (52%) create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClient.cs rename src/Libraries/Microsoft.Extensions.AI/Realtime/{LoggingRealtimeClientSessionBuilderExtensions.cs => LoggingRealtimeClientBuilderExtensions.cs} (62%) create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClient.cs rename src/Libraries/Microsoft.Extensions.AI/Realtime/{OpenTelemetryRealtimeClientSessionBuilderExtensions.cs => OpenTelemetryRealtimeClientBuilderExtensions.cs} (73%) create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientBuilder.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientBuilderRealtimeClientExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientExtensions.cs delete mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionBuilder.cs delete mode 100644 src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionBuilderRealtimeClientSessionExtensions.cs delete mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeClientSessionTests.cs rename test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/{FunctionInvokingRealtimeClientSessionTests.cs => FunctionInvokingRealtimeClientTests.cs} (79%) rename test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/{LoggingRealtimeClientSessionTests.cs => LoggingRealtimeClientTests.cs} (73%) rename test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/{OpenTelemetryRealtimeClientSessionTests.cs => OpenTelemetryRealtimeClientTests.cs} (85%) create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientBuilderTests.cs delete mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientSessionBuilderTests.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeClient.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeClient.cs new file mode 100644 index 00000000000..217f0851264 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeClient.cs @@ -0,0 +1,68 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Provides an optional base class for an that passes through calls to another instance. +/// +/// +/// This is recommended as a base type when building clients that can be chained around an underlying . +/// The default implementation simply passes each call to the inner client instance. +/// +[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] +public class DelegatingRealtimeClient : IRealtimeClient +{ + /// + /// Initializes a new instance of the class. + /// + /// The wrapped client instance. + /// is . + protected DelegatingRealtimeClient(IRealtimeClient innerClient) + { + InnerClient = Throw.IfNull(innerClient); + } + + /// + public void Dispose() + { + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + /// Gets the inner . + protected IRealtimeClient InnerClient { get; } + + /// + public virtual Task CreateSessionAsync( + RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default) => + InnerClient.CreateSessionAsync(options, cancellationToken); + + /// + public virtual object? GetService(Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(serviceType); + + // If the key is non-null, we don't know what it means so pass through to the inner service. + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + InnerClient.GetService(serviceType, serviceKey); + } + + /// Provides a mechanism for releasing unmanaged resources. + /// if being called from ; otherwise, . + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + InnerClient.Dispose(); + } + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeClientSession.cs deleted file mode 100644 index 44415a77181..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/DelegatingRealtimeClientSession.cs +++ /dev/null @@ -1,72 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Shared.DiagnosticIds; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Extensions.AI; - -/// -/// Provides an optional base class for an that passes through calls to another instance. -/// -/// -/// This is recommended as a base type when building sessions that can be chained around an underlying . -/// The default implementation simply passes each call to the inner session instance. -/// -[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class DelegatingRealtimeClientSession : IRealtimeClientSession -{ - /// - /// Initializes a new instance of the class. - /// - /// The wrapped session instance. - /// is . - protected DelegatingRealtimeClientSession(IRealtimeClientSession innerSession) - { - InnerSession = Throw.IfNull(innerSession); - } - - /// - public async ValueTask DisposeAsync() - { - await DisposeAsyncCore().ConfigureAwait(false); - GC.SuppressFinalize(this); - } - - /// Performs async cleanup of managed resources. - /// A task representing the asynchronous dispose operation. -#pragma warning disable EA0014 // The async method doesn't support cancellation - protected virtual ValueTask DisposeAsyncCore() => InnerSession.DisposeAsync(); -#pragma warning restore EA0014 - - /// Gets the inner . - protected IRealtimeClientSession InnerSession { get; } - - /// - public virtual RealtimeSessionOptions? Options => InnerSession.Options; - - /// - public virtual Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => - InnerSession.SendAsync(message, cancellationToken); - - /// - public virtual IAsyncEnumerable GetStreamingResponseAsync( - CancellationToken cancellationToken = default) => - InnerSession.GetStreamingResponseAsync(cancellationToken); - - /// - public virtual object? GetService(Type serviceType, object? serviceKey = null) - { - _ = Throw.IfNull(serviceType); - - // If the key is non-null, we don't know what it means so pass through to the inner service. - return - serviceKey is null && serviceType.IsInstanceOfType(this) ? this : - InnerSession.GetService(serviceType, serviceKey); - } -} diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClient.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClient.cs new file mode 100644 index 00000000000..ce1627086e5 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClient.cs @@ -0,0 +1,131 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// A delegating realtime client that invokes functions defined on . +/// Include this in a realtime client pipeline to resolve function calls automatically. +/// +/// +/// +/// When sessions created by this client receive a in a realtime server message from the inner +/// , they respond by invoking the corresponding defined +/// in (or in ), producing a +/// that is sent back to the inner session. This loop is repeated until there are no more function calls to make, or until +/// another stop condition is met, such as hitting . +/// +/// +[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class FunctionInvokingRealtimeClient : DelegatingRealtimeClient +{ + private readonly ILoggerFactory? _loggerFactory; + private readonly IServiceProvider? _services; + + /// + /// Initializes a new instance of the class. + /// + /// The inner . + /// An to use for logging information about function invocation. + /// An optional to use for resolving services required by the instances being invoked. + public FunctionInvokingRealtimeClient(IRealtimeClient innerClient, ILoggerFactory? loggerFactory = null, IServiceProvider? functionInvocationServices = null) + : base(innerClient) + { + _loggerFactory = loggerFactory; + _services = functionInvocationServices; + } + + /// + /// Gets the for the current function invocation. + /// + /// + /// This value flows across async calls. + /// + public static FunctionInvocationContext? CurrentContext => FunctionInvokingRealtimeClientSession.CurrentContext; + + /// + /// Gets or sets a value indicating whether detailed exception information should be included + /// in the response when calling the underlying . + /// + /// + /// if the full exception message is added to the response. + /// if a generic error message is included in the response. + /// The default value is . + /// + public bool IncludeDetailedErrors { get; set; } + + /// + /// Gets or sets a value indicating whether to allow concurrent invocation of functions. + /// + /// + /// if multiple function calls can execute in parallel. + /// if function calls are processed serially. + /// The default value is . + /// + public bool AllowConcurrentInvocation { get; set; } + + /// + /// Gets or sets the maximum number of iterations per request. + /// + /// + /// The maximum number of iterations per request. + /// The default value is 40. + /// + public int MaximumIterationsPerRequest + { + get; + set + { + if (value < 1) + { + Throw.ArgumentOutOfRangeException(nameof(value)); + } + + field = value; + } + } = 40; + + /// + /// Gets or sets the maximum number of consecutive iterations that are allowed to fail with an error. + /// + /// + /// The maximum number of consecutive iterations that are allowed to fail with an error. + /// The default value is 3. + /// + public int MaximumConsecutiveErrorsPerRequest + { + get; + set => field = Throw.IfLessThan(value, 0); + } = 3; + + /// Gets or sets a collection of additional tools the session is able to invoke. + public IList? AdditionalTools { get; set; } + + /// Gets or sets a value indicating whether a request to call an unknown function should terminate the function calling loop. + /// + /// to terminate the function calling loop and return the response if a request to call a tool + /// that isn't available is received; to create and send a + /// function result message stating that the tool couldn't be found. The default is . + /// + public bool TerminateOnUnknownCalls { get; set; } + + /// Gets or sets a delegate used to invoke instances. + public Func>? FunctionInvoker { get; set; } + + /// + public override async Task CreateSessionAsync( + RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default) + { + var innerSession = await base.CreateSessionAsync(options, cancellationToken).ConfigureAwait(false); + return new FunctionInvokingRealtimeClientSession(innerSession, this, _loggerFactory, _services); + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSessionBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientBuilderExtensions.cs similarity index 52% rename from src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSessionBuilderExtensions.cs rename to src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientBuilderExtensions.cs index ea7291080e2..06c90d7a104 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSessionBuilderExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientBuilderExtensions.cs @@ -5,39 +5,39 @@ using System.Diagnostics.CodeAnalysis; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Shared.DiagnosticIds; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; /// -/// Provides extension methods for attaching a to a realtime session pipeline. +/// Provides extension methods for attaching function invocation middleware to a realtime client pipeline. /// -[Experimental("MEAI001")] -public static class FunctionInvokingRealtimeClientSessionBuilderExtensions +[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] +public static class FunctionInvokingRealtimeClientBuilderExtensions { /// - /// Enables automatic function call invocation on the realtime session pipeline. + /// Enables automatic function call invocation on the realtime client pipeline. /// - /// This works by adding an instance of with default options. - /// The being used to build the realtime session pipeline. + /// The being used to build the realtime client pipeline. /// An optional to use to create a logger for logging function invocations. - /// An optional callback that can be used to configure the instance. + /// An optional callback that can be used to configure the instance. /// The supplied . /// is . - public static RealtimeClientSessionBuilder UseFunctionInvocation( - this RealtimeClientSessionBuilder builder, + public static RealtimeClientBuilder UseFunctionInvocation( + this RealtimeClientBuilder builder, ILoggerFactory? loggerFactory = null, - Action? configure = null) + Action? configure = null) { _ = Throw.IfNull(builder); - return builder.Use((innerSession, services) => + return builder.Use((innerClient, services) => { loggerFactory ??= services.GetService(); - var realtimeSession = new FunctionInvokingRealtimeClientSession(innerSession, loggerFactory, services); - configure?.Invoke(realtimeSession); - return realtimeSession; + var client = new FunctionInvokingRealtimeClient(innerClient, loggerFactory, services); + configure?.Invoke(client); + return client; }); } } diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs index 1133777b4fa..14704b48f8a 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -48,14 +47,13 @@ namespace Microsoft.Extensions.AI; /// tools being used concurrently (one per request). /// /// -[Experimental("MEAI001")] -public class FunctionInvokingRealtimeClientSession : DelegatingRealtimeClientSession +internal sealed class FunctionInvokingRealtimeClientSession : IRealtimeClientSession { /// The for the current function invocation. private static readonly AsyncLocal _currentContext = new(); /// Gets the specified when constructing the , if any. - protected IServiceProvider? FunctionInvocationServices { get; } + private IServiceProvider? FunctionInvocationServices { get; } /// The logger to use for logging information about function invocation. private readonly ILogger _logger; @@ -64,15 +62,23 @@ public class FunctionInvokingRealtimeClientSession : DelegatingRealtimeClientSes /// This component does not own the instance and should not dispose it. private readonly ActivitySource? _activitySource; + /// The inner session to delegate to. + private readonly IRealtimeClientSession _innerSession; + + /// The owning client that holds configuration. + private readonly FunctionInvokingRealtimeClient _client; + /// /// Initializes a new instance of the class. /// /// The underlying , or the next instance in a chain of sessions. + /// The owning that holds configuration. /// An to use for logging information about function invocation. /// An optional to use for resolving services required by the instances being invoked. - public FunctionInvokingRealtimeClientSession(IRealtimeClientSession innerSession, ILoggerFactory? loggerFactory = null, IServiceProvider? functionInvocationServices = null) - : base(innerSession) + public FunctionInvokingRealtimeClientSession(IRealtimeClientSession innerSession, FunctionInvokingRealtimeClient client, ILoggerFactory? loggerFactory = null, IServiceProvider? functionInvocationServices = null) { + _innerSession = Throw.IfNull(innerSession); + _client = Throw.IfNull(client); _logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; _activitySource = innerSession.GetService(); FunctionInvocationServices = functionInvocationServices; @@ -90,166 +96,51 @@ public FunctionInvokingRealtimeClientSession(IRealtimeClientSession innerSession /// /// This value flows across async calls. /// - public static FunctionInvocationContext? CurrentContext + internal static FunctionInvocationContext? CurrentContext { get => _currentContext.Value; - protected set => _currentContext.Value = value; + set => _currentContext.Value = value; } - /// - /// Gets or sets a value indicating whether detailed exception information should be included - /// in the response when calling the underlying . - /// - /// - /// if the full exception message is added to the response - /// when calling the underlying . - /// if a generic error message is included in the response. - /// The default value is . - /// - /// - /// - /// Setting the value to prevents the underlying model from disclosing - /// raw exception details to the end user, since it doesn't receive that information. Even in this - /// case, the raw object is available to application code by inspecting - /// the property. - /// - /// - /// Setting the value to can help the underlying bypass problems on - /// its own, for example by retrying the function call with different arguments. However it might - /// result in disclosing the raw exception information to external users, which can be a security - /// concern depending on the application scenario. - /// - /// - /// Changing the value of this property while the session is in use might result in inconsistencies - /// as to whether detailed errors are provided during an in-flight request. - /// - /// - public bool IncludeDetailedErrors { get; set; } + private bool IncludeDetailedErrors => _client.IncludeDetailedErrors; - /// - /// Gets or sets a value indicating whether to allow concurrent invocation of functions. - /// - /// - /// if multiple function calls can execute in parallel. - /// if function calls are processed serially. - /// The default value is . - /// - /// - /// An individual response from the inner session might contain multiple function call requests. - /// By default, such function calls are processed serially. Set to - /// to enable concurrent invocation such that multiple function calls can execute in parallel. - /// - public bool AllowConcurrentInvocation { get; set; } + private bool AllowConcurrentInvocation => _client.AllowConcurrentInvocation; - /// - /// Gets or sets the maximum number of iterations per request. - /// - /// - /// The maximum number of iterations per request. - /// The default value is 40. - /// - /// - /// - /// Each streaming request to this might end up making - /// multiple function call invocations. Each time the inner session responds with - /// a function call request, this session might perform that invocation and send the results - /// back to the inner session. This property limits the number of times - /// such an invocation is performed. - /// - /// - /// Changing the value of this property while the session is in use might result in inconsistencies - /// as to how many iterations are allowed for an in-flight request. - /// - /// - public int MaximumIterationsPerRequest - { - get; - set - { - if (value < 1) - { - Throw.ArgumentOutOfRangeException(nameof(value)); - } + private int MaximumIterationsPerRequest => _client.MaximumIterationsPerRequest; - field = value; - } - } = 40; + private int MaximumConsecutiveErrorsPerRequest => _client.MaximumConsecutiveErrorsPerRequest; - /// - /// Gets or sets the maximum number of consecutive iterations that are allowed to fail with an error. - /// - /// - /// The maximum number of consecutive iterations that are allowed to fail with an error. - /// The default value is 3. - /// - /// - /// - /// When function invocations fail with an exception, the - /// continues to send responses to the inner session, optionally supplying exception information (as - /// controlled by ). This allows the to - /// recover from errors by trying other function parameters that might succeed. - /// - /// - /// However, in case function invocations continue to produce exceptions, this property can be used to - /// limit the number of consecutive failing attempts. When the limit is reached, the exception will be - /// rethrown to the caller. - /// - /// - /// If the value is set to zero, all function calling exceptions immediately terminate the function - /// invocation loop and the exception will be rethrown to the caller. - /// - /// - /// Changing the value of this property while the session is in use might result in inconsistencies - /// as to how many iterations are allowed for an in-flight request. - /// - /// - public int MaximumConsecutiveErrorsPerRequest + private IList? AdditionalTools => _client.AdditionalTools; + + private bool TerminateOnUnknownCalls => _client.TerminateOnUnknownCalls; + + private Func>? FunctionInvoker => _client.FunctionInvoker; + + /// + public RealtimeSessionOptions? Options => _innerSession.Options; + + /// + public Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => + _innerSession.SendAsync(message, cancellationToken); + + /// + public object? GetService(Type serviceType, object? serviceKey = null) { - get; - set => field = Throw.IfLessThan(value, 0); - } = 3; + _ = Throw.IfNull(serviceType); - /// Gets or sets a collection of additional tools the session is able to invoke. - /// - /// These will not impact the requests sent by the , which will pass through the - /// unmodified. However, if the inner session requests the invocation of a tool - /// that was not in , this collection will also be consulted - /// to look for a corresponding tool to invoke. This is useful when the service might have been preconfigured to be aware - /// of certain tools that aren't also sent on each individual request. - /// - public IList? AdditionalTools { get; set; } - - /// Gets or sets a value indicating whether a request to call an unknown function should terminate the function calling loop. - /// - /// to terminate the function calling loop and return the response if a request to call a tool - /// that isn't available to the is received; to create and send a - /// function result message to the inner session stating that the tool couldn't be found. The default is . - /// - /// - /// - /// When , call requests to any tools that aren't available to the - /// will result in a response message automatically being created and returned to the inner session stating that the tool couldn't be - /// found. This behavior can help in cases where a model hallucinates a function, but it's problematic if the model has been made aware - /// of the existence of tools outside of the normal mechanisms, and requests one of those. can be used - /// to help with that. But if instead the consumer wants to know about all function call requests that the session can't handle, - /// can be set to . Upon receiving a request to call a function - /// that the doesn't know about, it will terminate the function calling loop and return - /// the response, leaving the handling of the function call requests to the consumer of the session. - /// - /// - /// s that the is aware of (for example, because they're in - /// or ) but that aren't s aren't considered - /// unknown, just not invocable. Any requests to a non-invocable tool will also result in the function calling loop terminating, - /// regardless of . - /// - /// - public bool TerminateOnUnknownCalls { get; set; } + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + _innerSession.GetService(serviceType, serviceKey); + } - /// Gets or sets a delegate used to invoke instances. - public Func>? FunctionInvoker { get; set; } + /// + public async ValueTask DisposeAsync() + { + await _innerSession.DisposeAsync().ConfigureAwait(false); + } /// - public override async IAsyncEnumerable GetStreamingResponseAsync( + public async IAsyncEnumerable GetStreamingResponseAsync( [EnumeratorCancellation] CancellationToken cancellationToken = default) { // Create an activity to group function invocations together for better observability. @@ -260,7 +151,7 @@ public override async IAsyncEnumerable GetStreamingRespon int consecutiveErrorCount = 0; int iterationCount = 0; - await foreach (var message in InnerSession.GetStreamingResponseAsync(cancellationToken).ConfigureAwait(false)) + await foreach (var message in _innerSession.GetStreamingResponseAsync(cancellationToken).ConfigureAwait(false)) { // Check if this message contains function calls bool hasFunctionCalls = false; @@ -305,7 +196,7 @@ public override async IAsyncEnumerable GetStreamingRespon foreach (var resultMessage in results.functionResults) { // inject back the function result messages to the inner session - await InnerSession.SendAsync(resultMessage, cancellationToken).ConfigureAwait(false); + await _innerSession.SendAsync(resultMessage, cancellationToken).ConfigureAwait(false); } } } @@ -341,7 +232,7 @@ private static bool ExtractFunctionCalls(RealtimeServerResponseOutputItemMessage /// private bool ShouldTerminateBasedOnFunctionCalls(List functionCallContents) { - var (toolMap, _) = FunctionInvocationHelpers.CreateToolsMap(AdditionalTools, InnerSession.Options?.Tools as IList); + var (toolMap, _) = FunctionInvocationHelpers.CreateToolsMap(AdditionalTools, _innerSession.Options?.Tools as IList); if (toolMap is null || toolMap.Count == 0) { @@ -389,7 +280,7 @@ private bool ShouldTerminateBasedOnFunctionCalls(List funct CancellationToken cancellationToken) { // Compute toolMap to ensure we always use the latest tools - var (toolMap, _) = FunctionInvocationHelpers.CreateToolsMap(AdditionalTools, InnerSession.Options?.Tools as IList); + var (toolMap, _) = FunctionInvocationHelpers.CreateToolsMap(AdditionalTools, _innerSession.Options?.Tools as IList); var captureCurrentIterationExceptions = consecutiveErrorCount < MaximumConsecutiveErrorsPerRequest; @@ -474,7 +365,7 @@ private List CreateFunctionResultMessages(ListThe function invocation context. /// Cancellation token. /// The function result. - protected virtual ValueTask InvokeFunctionAsync(FunctionInvocationContext context, CancellationToken cancellationToken) + private ValueTask InvokeFunctionAsync(FunctionInvocationContext context, CancellationToken cancellationToken) { _ = Throw.IfNull(context); diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClient.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClient.cs new file mode 100644 index 00000000000..e1b1bb92915 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClient.cs @@ -0,0 +1,56 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// A delegating realtime client that logs operations to an . +/// +/// +/// When the employed enables , the contents of +/// messages and options are logged. These messages and options may contain sensitive application data. +/// is disabled by default and should never be enabled in a production environment. +/// Messages and options are not logged at other logging levels. +/// +/// +[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class LoggingRealtimeClient : DelegatingRealtimeClient +{ + private readonly ILogger _logger; + private JsonSerializerOptions _jsonSerializerOptions; + + /// Initializes a new instance of the class. + /// The inner . + /// An instance that will be used for all logging. + public LoggingRealtimeClient(IRealtimeClient innerClient, ILogger logger) + : base(innerClient) + { + _logger = Throw.IfNull(logger); + _jsonSerializerOptions = AIJsonUtilities.DefaultOptions; + } + + /// Gets or sets JSON serialization options to use when serializing logging data. + public JsonSerializerOptions JsonSerializerOptions + { + get => _jsonSerializerOptions; + set => _jsonSerializerOptions = Throw.IfNull(value); + } + + /// + public override async Task CreateSessionAsync( + RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default) + { + var innerSession = await base.CreateSessionAsync(options, cancellationToken).ConfigureAwait(false); + return new LoggingRealtimeClientSession(innerSession, _logger) + { + JsonSerializerOptions = _jsonSerializerOptions, + }; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClientSessionBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClientBuilderExtensions.cs similarity index 62% rename from src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClientSessionBuilderExtensions.cs rename to src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClientBuilderExtensions.cs index f22dbcbfbe3..a3aedf33b4c 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClientSessionBuilderExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClientBuilderExtensions.cs @@ -6,21 +6,22 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Shared.DiagnosticIds; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; -/// Provides extensions for configuring instances. -[Experimental("MEAI001")] -public static class LoggingRealtimeClientSessionBuilderExtensions +/// Provides extensions for configuring logging on an pipeline. +[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] +public static class LoggingRealtimeClientBuilderExtensions { - /// Adds logging to the realtime session pipeline. - /// The . + /// Adds logging to the realtime client pipeline. + /// The . /// /// An optional used to create a logger with which logging should be performed. /// If not supplied, a required instance will be resolved from the service provider. /// - /// An optional callback that can be used to configure the instance. + /// An optional callback that can be used to configure the instance. /// The . /// is . /// @@ -31,27 +32,28 @@ public static class LoggingRealtimeClientSessionBuilderExtensions /// Messages and options are not logged at other logging levels. /// /// - public static RealtimeClientSessionBuilder UseLogging( - this RealtimeClientSessionBuilder builder, + public static RealtimeClientBuilder UseLogging( + this RealtimeClientBuilder builder, ILoggerFactory? loggerFactory = null, - Action? configure = null) + Action? configure = null) { _ = Throw.IfNull(builder); - return builder.Use((innerSession, services) => + return builder.Use((innerClient, services) => { loggerFactory ??= services.GetRequiredService(); - // If the factory we resolve is for the null logger, the LoggingRealtimeClientSession will end up - // being an expensive nop, so skip adding it and just return the inner session. + // If the factory we resolve is for the null logger, the LoggingRealtimeClient will end up + // being an expensive nop, so skip adding it and just return the inner client. if (loggerFactory == NullLoggerFactory.Instance) { - return innerSession; + return innerClient; } - var session = new LoggingRealtimeClientSession(innerSession, loggerFactory.CreateLogger(typeof(LoggingRealtimeClientSession))); - configure?.Invoke(session); - return session; + var logger = loggerFactory.CreateLogger(typeof(LoggingRealtimeClient)); + var client = new LoggingRealtimeClient(innerClient, logger); + configure?.Invoke(client); + return client; }); } } diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClientSession.cs index 3b801c54c64..e1cf3b88a97 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/LoggingRealtimeClientSession.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Text.Json; using System.Text.Json.Nodes; @@ -27,12 +26,14 @@ namespace Microsoft.Extensions.AI; /// Messages and options are not logged at other logging levels. /// /// -[Experimental("MEAI001")] -public partial class LoggingRealtimeClientSession : DelegatingRealtimeClientSession +internal sealed partial class LoggingRealtimeClientSession : IRealtimeClientSession { /// An instance used for all logging. private readonly ILogger _logger; + /// The inner session to delegate to. + private readonly IRealtimeClientSession _innerSession; + /// The to use for serialization of state written to the logger. private JsonSerializerOptions _jsonSerializerOptions; @@ -40,12 +41,15 @@ public partial class LoggingRealtimeClientSession : DelegatingRealtimeClientSess /// The underlying . /// An instance that will be used for all logging. public LoggingRealtimeClientSession(IRealtimeClientSession innerSession, ILogger logger) - : base(innerSession) { + _innerSession = Throw.IfNull(innerSession); _logger = Throw.IfNull(logger); _jsonSerializerOptions = AIJsonUtilities.DefaultOptions; } + /// + public RealtimeSessionOptions? Options => _innerSession.Options; + /// Gets or sets JSON serialization options to use when serializing logging data. public JsonSerializerOptions JsonSerializerOptions { @@ -53,8 +57,24 @@ public JsonSerializerOptions JsonSerializerOptions set => _jsonSerializerOptions = Throw.IfNull(value); } + /// + public async ValueTask DisposeAsync() + { + await _innerSession.DisposeAsync().ConfigureAwait(false); + } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + _innerSession.GetService(serviceType, serviceKey); + } + /// - public override async Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) + public async Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) { _ = Throw.IfNull(message); @@ -72,7 +92,7 @@ public override async Task SendAsync(RealtimeClientMessage message, Cancellation try { - await base.SendAsync(message, cancellationToken).ConfigureAwait(false); + await _innerSession.SendAsync(message, cancellationToken).ConfigureAwait(false); if (_logger.IsEnabled(LogLevel.Debug)) { @@ -92,7 +112,7 @@ public override async Task SendAsync(RealtimeClientMessage message, Cancellation } /// - public override async IAsyncEnumerable GetStreamingResponseAsync( + public async IAsyncEnumerable GetStreamingResponseAsync( [EnumeratorCancellation] CancellationToken cancellationToken = default) { if (_logger.IsEnabled(LogLevel.Debug)) @@ -103,7 +123,7 @@ public override async IAsyncEnumerable GetStreamingRespon IAsyncEnumerator e; try { - e = base.GetStreamingResponseAsync(cancellationToken).GetAsyncEnumerator(cancellationToken); + e = _innerSession.GetStreamingResponseAsync(cancellationToken).GetAsyncEnumerator(cancellationToken); } catch (OperationCanceledException) { diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClient.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClient.cs new file mode 100644 index 00000000000..a0e4510d5c4 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClient.cs @@ -0,0 +1,71 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// A delegating realtime client that adds OpenTelemetry support, following the OpenTelemetry Semantic Conventions for Generative AI systems. +/// +/// +/// +/// The draft specification this follows is available at . +/// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. +/// +/// +[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class OpenTelemetryRealtimeClient : DelegatingRealtimeClient +{ + private readonly ILogger? _logger; + private readonly string? _sourceName; + private JsonSerializerOptions _jsonSerializerOptions; + + /// Initializes a new instance of the class. + /// The inner . + /// The to use for emitting any logging data from the client. + /// An optional source name that will be used on the telemetry data. + public OpenTelemetryRealtimeClient(IRealtimeClient innerClient, ILogger? logger = null, string? sourceName = null) + : base(innerClient) + { + _logger = logger; + _sourceName = sourceName; + _jsonSerializerOptions = AIJsonUtilities.DefaultOptions; + } + + /// + /// Gets or sets a value indicating whether potentially sensitive information should be included in telemetry. + /// + /// + /// if potentially sensitive information should be included in telemetry; + /// if telemetry shouldn't include raw inputs and outputs. + /// The default value is , unless the OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT + /// environment variable is set to "true" (case-insensitive). + /// + public bool EnableSensitiveData { get; set; } = TelemetryHelpers.EnableSensitiveDataDefault; + + /// Gets or sets JSON serialization options to use when formatting realtime data into telemetry strings. + public JsonSerializerOptions JsonSerializerOptions + { + get => _jsonSerializerOptions; + set => _jsonSerializerOptions = Throw.IfNull(value); + } + + /// + public override async Task CreateSessionAsync( + RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default) + { + var innerSession = await base.CreateSessionAsync(options, cancellationToken).ConfigureAwait(false); + return new OpenTelemetryRealtimeClientSession(innerSession, _logger, _sourceName) + { + EnableSensitiveData = EnableSensitiveData, + JsonSerializerOptions = _jsonSerializerOptions, + }; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSessionBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientBuilderExtensions.cs similarity index 73% rename from src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSessionBuilderExtensions.cs rename to src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientBuilderExtensions.cs index eb69e12c1ac..cd17b03bfb2 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSessionBuilderExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientBuilderExtensions.cs @@ -5,21 +5,22 @@ using System.Diagnostics.CodeAnalysis; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Shared.DiagnosticIds; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; -/// Provides extensions for configuring instances. -[Experimental("MEAI001")] -public static class OpenTelemetryRealtimeClientSessionBuilderExtensions +/// Provides extensions for configuring OpenTelemetry on an pipeline. +[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] +public static class OpenTelemetryRealtimeClientBuilderExtensions { /// - /// Adds OpenTelemetry support to the realtime session pipeline, following the OpenTelemetry Semantic Conventions for Generative AI systems. + /// Adds OpenTelemetry support to the realtime client pipeline, following the OpenTelemetry Semantic Conventions for Generative AI systems. /// /// /// /// The draft specification this follows is available at . - /// The specification is still experimental and subject to change; as such, the telemetry output by this session is also subject to change. + /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// /// /// The following standard OpenTelemetry GenAI conventions are supported: @@ -55,24 +56,24 @@ public static class OpenTelemetryRealtimeClientSessionBuilderExtensions /// /// /// - /// The . + /// The . /// An optional to use to create a logger for logging events. /// An optional source name that will be used on the telemetry data. - /// An optional callback that can be used to configure the instance. + /// An optional callback that can be used to configure the instance. /// The . /// is . - public static RealtimeClientSessionBuilder UseOpenTelemetry( - this RealtimeClientSessionBuilder builder, + public static RealtimeClientBuilder UseOpenTelemetry( + this RealtimeClientBuilder builder, ILoggerFactory? loggerFactory = null, string? sourceName = null, - Action? configure = null) => - Throw.IfNull(builder).Use((innerSession, services) => + Action? configure = null) => + Throw.IfNull(builder).Use((innerClient, services) => { loggerFactory ??= services.GetService(); - var session = new OpenTelemetryRealtimeClientSession(innerSession, loggerFactory?.CreateLogger(typeof(OpenTelemetryRealtimeClientSession)), sourceName); - configure?.Invoke(session); - - return session; + var logger = loggerFactory?.CreateLogger(typeof(OpenTelemetryRealtimeClient)); + var client = new OpenTelemetryRealtimeClient(innerClient, logger, sourceName); + configure?.Invoke(client); + return client; }); } diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs index 875ca61ba18..81e40f081ee 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; using System.Diagnostics.Metrics; using System.Linq; using System.Runtime.CompilerServices; @@ -75,8 +74,7 @@ namespace Microsoft.Extensions.AI; /// /// /// -[Experimental("MEAI001")] -public sealed partial class OpenTelemetryRealtimeClientSession : DelegatingRealtimeClientSession +internal sealed partial class OpenTelemetryRealtimeClientSession : IRealtimeClientSession { private readonly ActivitySource _activitySource; private readonly Meter _meter; @@ -89,6 +87,8 @@ public sealed partial class OpenTelemetryRealtimeClientSession : DelegatingRealt private readonly string? _serverAddress; private readonly int _serverPort; + private readonly IRealtimeClientSession _innerSession; + private JsonSerializerOptions _jsonSerializerOptions; /// Initializes a new instance of the class. @@ -98,12 +98,11 @@ public sealed partial class OpenTelemetryRealtimeClientSession : DelegatingRealt #pragma warning disable IDE0060 // Remove unused parameter; it exists for backwards compatibility and future use public OpenTelemetryRealtimeClientSession(IRealtimeClientSession innerSession, ILogger? logger = null, string? sourceName = null) #pragma warning restore IDE0060 - : base(innerSession) { - Debug.Assert(innerSession is not null, "Should have been validated by the base ctor"); + _innerSession = Throw.IfNull(innerSession); // Try to get metadata from the inner session's ChatClientMetadata if available - if (innerSession!.GetService(typeof(ChatClientMetadata)) is ChatClientMetadata metadata) + if (innerSession.GetService(typeof(ChatClientMetadata)) is ChatClientMetadata metadata) { _defaultModelId = metadata.DefaultModelId; _providerName = metadata.ProviderName; @@ -139,12 +138,15 @@ public JsonSerializerOptions JsonSerializerOptions set => _jsonSerializerOptions = Throw.IfNull(value); } - /// - protected override async ValueTask DisposeAsyncCore() + /// + public RealtimeSessionOptions? Options => _innerSession.Options; + + /// + public async ValueTask DisposeAsync() { _activitySource.Dispose(); _meter.Dispose(); - await base.DisposeAsyncCore().ConfigureAwait(false); + await _innerSession.DisposeAsync().ConfigureAwait(false); } /// @@ -165,12 +167,18 @@ protected override async ValueTask DisposeAsyncCore() public bool EnableSensitiveData { get; set; } = TelemetryHelpers.EnableSensitiveDataDefault; /// - public override object? GetService(Type serviceType, object? serviceKey = null) => - serviceType == typeof(ActivitySource) ? _activitySource : - base.GetService(serviceType, serviceKey); + public object? GetService(Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(serviceType); + + return + serviceType == typeof(ActivitySource) ? _activitySource : + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + _innerSession.GetService(serviceType, serviceKey); + } /// - public override async Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) + public async Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) { if (EnableSensitiveData && _activitySource.HasListeners()) { @@ -186,11 +194,11 @@ public override async Task SendAsync(RealtimeClientMessage message, Cancellation } } - await base.SendAsync(message, cancellationToken).ConfigureAwait(false); + await _innerSession.SendAsync(message, cancellationToken).ConfigureAwait(false); } /// - public override async IAsyncEnumerable GetStreamingResponseAsync( + public async IAsyncEnumerable GetStreamingResponseAsync( [EnumeratorCancellation] CancellationToken cancellationToken = default) { _jsonSerializerOptions.MakeReadOnly(); @@ -207,7 +215,7 @@ public override async IAsyncEnumerable GetStreamingRespon IAsyncEnumerable responses; try { - responses = base.GetStreamingResponseAsync(cancellationToken); + responses = _innerSession.GetStreamingResponseAsync(cancellationToken); } catch (Exception ex) { diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientBuilder.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientBuilder.cs new file mode 100644 index 00000000000..ce42d18e027 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientBuilder.cs @@ -0,0 +1,89 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// A builder for creating pipelines of . +[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class RealtimeClientBuilder +{ + private readonly Func _innerClientFactory; + + /// The registered client factory instances. + private List>? _clientFactories; + + /// Initializes a new instance of the class. + /// The inner that represents the underlying backend. + /// is . + public RealtimeClientBuilder(IRealtimeClient innerClient) + { + _ = Throw.IfNull(innerClient); + _innerClientFactory = _ => innerClient; + } + + /// Initializes a new instance of the class. + /// A callback that produces the inner that represents the underlying backend. + public RealtimeClientBuilder(Func innerClientFactory) + { + _innerClientFactory = Throw.IfNull(innerClientFactory); + } + + /// Builds an that represents the entire pipeline. Calls to this instance will pass through each of the pipeline stages in turn. + /// + /// The that should provide services to the instances. + /// If , an empty will be used. + /// + /// An instance of that represents the entire pipeline. + public IRealtimeClient Build(IServiceProvider? services = null) + { + services ??= EmptyServiceProvider.Instance; + var client = _innerClientFactory(services); + + // To match intuitive expectations, apply the factories in reverse order, so that the first factory added is the outermost. + if (_clientFactories is not null) + { + for (var i = _clientFactories.Count - 1; i >= 0; i--) + { + client = _clientFactories[i](client, services); + if (client is null) + { + Throw.InvalidOperationException( + $"The {nameof(RealtimeClientBuilder)} entry at index {i} returned null. " + + $"Ensure that the callbacks passed to {nameof(Use)} return non-null {nameof(IRealtimeClient)} instances."); + } + } + } + + return client; + } + + /// Adds a factory for an intermediate realtime client to the realtime client pipeline. + /// The client factory function. + /// The updated instance. + /// is . + public RealtimeClientBuilder Use(Func clientFactory) + { + _ = Throw.IfNull(clientFactory); + + return Use((innerClient, _) => clientFactory(innerClient)); + } + + /// Adds a factory for an intermediate realtime client to the realtime client pipeline. + /// The client factory function. + /// The updated instance. + /// is . + public RealtimeClientBuilder Use(Func clientFactory) + { + _ = Throw.IfNull(clientFactory); + + (_clientFactories ??= []).Add(clientFactory); + return this; + } + +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientBuilderRealtimeClientExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientBuilderRealtimeClientExtensions.cs new file mode 100644 index 00000000000..7be2a697059 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientBuilderRealtimeClientExtensions.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Provides extension methods for working with in the context of . +[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] +public static class RealtimeClientBuilderRealtimeClientExtensions +{ + /// Creates a new using as its inner client. + /// The client to use as the inner client. + /// The new instance. + /// + /// This method is equivalent to using the constructor directly, + /// specifying as the inner client. + /// + /// is . + public static RealtimeClientBuilder AsBuilder(this IRealtimeClient innerClient) + { + _ = Throw.IfNull(innerClient); + + return new RealtimeClientBuilder(innerClient); + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientExtensions.cs new file mode 100644 index 00000000000..2efca1329f3 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientExtensions.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Provides a collection of static methods for extending instances. +[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] +public static class RealtimeClientExtensions +{ + /// Asks the for an object of type . + /// The type of the object to be retrieved. + /// The client. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly typed services that may be provided by the , + /// including itself or any services it might be wrapping. + /// + public static TService? GetService(this IRealtimeClient client, object? serviceKey = null) + { + _ = Throw.IfNull(client); + + return client.GetService(typeof(TService), serviceKey) is TService service ? service : default; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionBuilder.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionBuilder.cs deleted file mode 100644 index f5ed69f2a21..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionBuilder.cs +++ /dev/null @@ -1,88 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Extensions.AI; - -/// A builder for creating pipelines of . -[Experimental("MEAI001")] -public sealed class RealtimeClientSessionBuilder -{ - private readonly Func _innerSessionFactory; - - /// The registered session factory instances. - private List>? _sessionFactories; - - /// Initializes a new instance of the class. - /// The inner that represents the underlying backend. - /// is . - public RealtimeClientSessionBuilder(IRealtimeClientSession innerSession) - { - _ = Throw.IfNull(innerSession); - _innerSessionFactory = _ => innerSession; - } - - /// Initializes a new instance of the class. - /// A callback that produces the inner that represents the underlying backend. - public RealtimeClientSessionBuilder(Func innerSessionFactory) - { - _innerSessionFactory = Throw.IfNull(innerSessionFactory); - } - - /// Builds an that represents the entire pipeline. Calls to this instance will pass through each of the pipeline stages in turn. - /// - /// The that should provide services to the instances. - /// If , an empty will be used. - /// - /// An instance of that represents the entire pipeline. - public IRealtimeClientSession Build(IServiceProvider? services = null) - { - services ??= EmptyServiceProvider.Instance; - var session = _innerSessionFactory(services); - - // To match intuitive expectations, apply the factories in reverse order, so that the first factory added is the outermost. - if (_sessionFactories is not null) - { - for (var i = _sessionFactories.Count - 1; i >= 0; i--) - { - session = _sessionFactories[i](session, services); - if (session is null) - { - Throw.InvalidOperationException( - $"The {nameof(RealtimeClientSessionBuilder)} entry at index {i} returned null. " + - $"Ensure that the callbacks passed to {nameof(Use)} return non-null {nameof(IRealtimeClientSession)} instances."); - } - } - } - - return session; - } - - /// Adds a factory for an intermediate realtime session to the realtime session pipeline. - /// The session factory function. - /// The updated instance. - /// is . - public RealtimeClientSessionBuilder Use(Func sessionFactory) - { - _ = Throw.IfNull(sessionFactory); - - return Use((innerSession, _) => sessionFactory(innerSession)); - } - - /// Adds a factory for an intermediate realtime session to the realtime session pipeline. - /// The session factory function. - /// The updated instance. - /// is . - public RealtimeClientSessionBuilder Use(Func sessionFactory) - { - _ = Throw.IfNull(sessionFactory); - - (_sessionFactories ??= []).Add(sessionFactory); - return this; - } - -} diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionBuilderRealtimeClientSessionExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionBuilderRealtimeClientSessionExtensions.cs deleted file mode 100644 index 84def11121d..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionBuilderRealtimeClientSessionExtensions.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Diagnostics.CodeAnalysis; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Extensions.AI; - -/// Provides extension methods for working with in the context of . -[Experimental("MEAI001")] -public static class RealtimeClientSessionBuilderRealtimeClientSessionExtensions -{ - /// Creates a new using as its inner session. - /// The session to use as the inner session. - /// The new instance. - /// - /// This method is equivalent to using the constructor directly, - /// specifying as the inner session. - /// - /// is . - public static RealtimeClientSessionBuilder AsBuilder(this IRealtimeClientSession innerSession) - { - _ = Throw.IfNull(innerSession); - - return new RealtimeClientSessionBuilder(innerSession); - } -} diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionExtensions.cs index c9a074c78e5..5e71973d4fb 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionExtensions.cs @@ -3,12 +3,13 @@ using System; using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; /// Provides a collection of static methods for extending instances. -[Experimental("MEAI001")] +[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] public static class RealtimeClientSessionExtensions { /// Asks the for an object of type . diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeClientSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeClientSessionTests.cs deleted file mode 100644 index 3ab43404473..00000000000 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/DelegatingRealtimeClientSessionTests.cs +++ /dev/null @@ -1,240 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; -using Xunit; - -#pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. - -namespace Microsoft.Extensions.AI; - -public class DelegatingRealtimeClientSessionTests -{ - [Fact] - public void Ctor_NullInnerSession_Throws() - { - Assert.Throws("innerSession", () => new NoOpDelegatingRealtimeClientSession(null!)); - } - - [Fact] - public async Task Options_DelegatesToInner() - { - var expectedOptions = new RealtimeSessionOptions { Model = "test-model" }; - await using var inner = new TestRealtimeClientSession { Options = expectedOptions }; - await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); - - Assert.Same(expectedOptions, delegating.Options); - } - - [Fact] - public async Task SendAsync_SessionUpdateMessage_DelegatesToInner() - { - var called = false; - var sentOptions = new RealtimeSessionOptions { Instructions = "Be helpful" }; - await using var inner = new TestRealtimeClientSession - { - SendAsyncCallback = (msg, _) => - { - var updateMsg = Assert.IsType(msg); - Assert.Same(sentOptions, updateMsg.Options); - called = true; - return Task.CompletedTask; - }, - }; - await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); - - await delegating.SendAsync(new RealtimeClientSessionUpdateMessage(sentOptions)); - Assert.True(called); - } - - [Fact] - public async Task SendAsync_DelegatesToInner() - { - var called = false; - var sentMessage = new RealtimeClientMessage { MessageId = "evt_001" }; - await using var inner = new TestRealtimeClientSession - { - SendAsyncCallback = (msg, _) => - { - Assert.Same(sentMessage, msg); - called = true; - return Task.CompletedTask; - }, - }; - await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); - - await delegating.SendAsync(sentMessage); - Assert.True(called); - } - - [Fact] - public async Task GetStreamingResponseAsync_DelegatesToInner() - { - var expected = new RealtimeServerMessage { Type = RealtimeServerMessageType.Error, MessageId = "evt_002" }; - await using var inner = new TestRealtimeClientSession - { - GetStreamingResponseAsyncCallback = (ct) => YieldSingle(expected, ct), - }; - await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); - - var messages = new List(); - await foreach (var msg in delegating.GetStreamingResponseAsync()) - { - messages.Add(msg); - } - - Assert.Single(messages); - Assert.Same(expected, messages[0]); - } - - [Fact] - public async Task GetService_ReturnsSelfForMatchingType() - { - await using var inner = new TestRealtimeClientSession(); - await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); - - Assert.Same(delegating, delegating.GetService(typeof(NoOpDelegatingRealtimeClientSession))); - Assert.Same(delegating, delegating.GetService(typeof(DelegatingRealtimeClientSession))); - Assert.Same(delegating, delegating.GetService(typeof(IRealtimeClientSession))); - } - - [Fact] - public async Task GetService_DelegatesToInnerForUnknownType() - { - await using var inner = new TestRealtimeClientSession(); - await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); - - // TestRealtimeClientSession returns itself for matching types - Assert.Same(inner, delegating.GetService(typeof(TestRealtimeClientSession))); - Assert.Null(delegating.GetService(typeof(string))); - } - - [Fact] - public async Task GetService_WithServiceKey_DelegatesToInner() - { - await using var inner = new TestRealtimeClientSession(); - await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); - - // With a non-null key, delegating should NOT return itself even for matching types - Assert.Null(delegating.GetService(typeof(NoOpDelegatingRealtimeClientSession), "someKey")); - } - - [Fact] - public async Task GetService_NullServiceType_Throws() - { - await using var inner = new TestRealtimeClientSession(); - await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); - - Assert.Throws("serviceType", () => delegating.GetService(null!)); - } - - [Fact] - public async Task DisposeAsync_DisposesInner() - { - var disposed = false; - await using var inner = new DisposableTestRealtimeClientSession(() => disposed = true); - var delegating = new NoOpDelegatingRealtimeClientSession(inner); - - await delegating.DisposeAsync(); - Assert.True(disposed); - } - - [Fact] - public async Task SendAsync_SessionUpdateMessage_FlowsCancellationToken() - { - CancellationToken capturedToken = default; - using var cts = new CancellationTokenSource(); - var sentOptions = new RealtimeSessionOptions(); - - await using var inner = new TestRealtimeClientSession - { - SendAsyncCallback = (msg, ct) => - { - capturedToken = ct; - return Task.CompletedTask; - }, - }; - await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); - - await delegating.SendAsync(new RealtimeClientSessionUpdateMessage(sentOptions), cts.Token); - Assert.Equal(cts.Token, capturedToken); - } - - [Fact] - public async Task SendAsync_FlowsCancellationToken() - { - CancellationToken capturedToken = default; - using var cts = new CancellationTokenSource(); - var sentMessage = new RealtimeClientMessage(); - - await using var inner = new TestRealtimeClientSession - { - SendAsyncCallback = (msg, ct) => - { - capturedToken = ct; - return Task.CompletedTask; - }, - }; - await using var delegating = new NoOpDelegatingRealtimeClientSession(inner); - - await delegating.SendAsync(sentMessage, cts.Token); - Assert.Equal(cts.Token, capturedToken); - } - - private static async IAsyncEnumerable YieldSingle( - RealtimeServerMessage message, - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - _ = cancellationToken; - await Task.CompletedTask.ConfigureAwait(false); - yield return message; - } - - /// A concrete DelegatingRealtimeClientSession for testing (since the base class is abstract-ish with protected ctor). - private sealed class NoOpDelegatingRealtimeClientSession : DelegatingRealtimeClientSession - { - public NoOpDelegatingRealtimeClientSession(IRealtimeClientSession innerSession) - : base(innerSession) - { - } - } - - /// A test session that tracks Dispose calls. - private sealed class DisposableTestRealtimeClientSession : IRealtimeClientSession - { - private readonly Action _onDispose; - - public DisposableTestRealtimeClientSession(Action onDispose) - { - _onDispose = onDispose; - } - - public RealtimeSessionOptions? Options => null; - - public Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default) => Task.CompletedTask; - - public IAsyncEnumerable GetStreamingResponseAsync( - CancellationToken cancellationToken = default) => - EmptyUpdatesServer(cancellationToken); - - public object? GetService(Type serviceType, object? serviceKey = null) => null; - - public ValueTask DisposeAsync() - { - _onDispose(); - return default; - } - - private static async IAsyncEnumerable EmptyUpdatesServer( - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - _ = cancellationToken; - await Task.CompletedTask.ConfigureAwait(false); - yield break; - } - } -} diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeClientSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeClientTests.cs similarity index 79% rename from test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeClientSessionTests.cs rename to test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeClientTests.cs index 0020a09f794..a3f661ad192 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeClientSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeClientTests.cs @@ -15,50 +15,47 @@ namespace Microsoft.Extensions.AI; -public class FunctionInvokingRealtimeClientSessionTests +public class FunctionInvokingRealtimeClientTests { [Fact] - public void Ctor_NullInnerSession_Throws() + public void Ctor_NullArgs_Throws() { - Assert.Throws("innerSession", () => new FunctionInvokingRealtimeClientSession(null!)); + Assert.Throws("innerClient", () => new FunctionInvokingRealtimeClient(null!)); } [Fact] - public async Task Properties_DefaultValues() + public void Properties_DefaultValues() { - await using var inner = new TestRealtimeClientSession(); - await using var session = new FunctionInvokingRealtimeClientSession(inner); - - Assert.False(session.IncludeDetailedErrors); - Assert.False(session.AllowConcurrentInvocation); - Assert.Equal(40, session.MaximumIterationsPerRequest); - Assert.Equal(3, session.MaximumConsecutiveErrorsPerRequest); - Assert.Null(session.AdditionalTools); - Assert.False(session.TerminateOnUnknownCalls); - Assert.Null(session.FunctionInvoker); + using var client = CreateClient(); + + Assert.False(client.IncludeDetailedErrors); + Assert.False(client.AllowConcurrentInvocation); + Assert.Equal(40, client.MaximumIterationsPerRequest); + Assert.Equal(3, client.MaximumConsecutiveErrorsPerRequest); + Assert.Null(client.AdditionalTools); + Assert.False(client.TerminateOnUnknownCalls); + Assert.Null(client.FunctionInvoker); } [Fact] - public async Task MaximumIterationsPerRequest_InvalidValue_Throws() + public void MaximumIterationsPerRequest_InvalidValue_Throws() { - await using var inner = new TestRealtimeClientSession(); - await using var session = new FunctionInvokingRealtimeClientSession(inner); + using var client = CreateClient(); - Assert.Throws("value", () => session.MaximumIterationsPerRequest = 0); - Assert.Throws("value", () => session.MaximumIterationsPerRequest = -1); + Assert.Throws("value", () => client.MaximumIterationsPerRequest = 0); + Assert.Throws("value", () => client.MaximumIterationsPerRequest = -1); } [Fact] - public async Task MaximumConsecutiveErrorsPerRequest_InvalidValue_Throws() + public void MaximumConsecutiveErrorsPerRequest_InvalidValue_Throws() { - await using var inner = new TestRealtimeClientSession(); - await using var session = new FunctionInvokingRealtimeClientSession(inner); + using var client = CreateClient(); - Assert.Throws("value", () => session.MaximumConsecutiveErrorsPerRequest = -1); + Assert.Throws("value", () => client.MaximumConsecutiveErrorsPerRequest = -1); // 0 is valid (means immediately rethrow on any error) - session.MaximumConsecutiveErrorsPerRequest = 0; - Assert.Equal(0, session.MaximumConsecutiveErrorsPerRequest); + client.MaximumConsecutiveErrorsPerRequest = 0; + Assert.Equal(0, client.MaximumConsecutiveErrorsPerRequest); } [Fact] @@ -74,7 +71,8 @@ public async Task GetStreamingResponseAsync_NoFunctionCalls_PassesThrough() { GetStreamingResponseAsyncCallback = (ct) => YieldMessages(serverMessages, ct), }; - await using var session = new FunctionInvokingRealtimeClientSession(inner); + using var client = CreateClient(inner); + await using var session = await client.CreateSessionAsync(); var received = new List(); await foreach (var msg in session.GetStreamingResponseAsync()) @@ -110,7 +108,8 @@ public async Task GetStreamingResponseAsync_FunctionCall_InvokesAndInjectsResult }, }; - await using var session = new FunctionInvokingRealtimeClientSession(inner); + using var client = CreateClient(inner); + await using var session = await client.CreateSessionAsync(); var received = new List(); await foreach (var msg in session.GetStreamingResponseAsync()) @@ -158,10 +157,9 @@ public async Task GetStreamingResponseAsync_FunctionCall_FromAdditionalTools() }, }; - await using var session = new FunctionInvokingRealtimeClientSession(inner) - { - AdditionalTools = [getWeather], - }; + using var client = CreateClient(inner); + client.AdditionalTools = [getWeather]; + await using var session = await client.CreateSessionAsync(); await foreach (var msg in session.GetStreamingResponseAsync()) { @@ -197,10 +195,9 @@ public async Task GetStreamingResponseAsync_MaxIterations_StopsInvoking() SendAsyncCallback = (_, _) => Task.CompletedTask, }; - await using var session = new FunctionInvokingRealtimeClientSession(inner) - { - MaximumIterationsPerRequest = 2, - }; + using var client = CreateClient(inner); + client.MaximumIterationsPerRequest = 2; + await using var session = await client.CreateSessionAsync(); var received = new List(); await foreach (var msg in session.GetStreamingResponseAsync()) @@ -234,14 +231,13 @@ public async Task GetStreamingResponseAsync_FunctionInvoker_CustomDelegate() SendAsyncCallback = (_, _) => Task.CompletedTask, }; - await using var session = new FunctionInvokingRealtimeClientSession(inner) + using var client = CreateClient(inner); + client.FunctionInvoker = (context, ct) => { - FunctionInvoker = (context, ct) => - { - customInvoked = true; - return new ValueTask("custom_result"); - }, + customInvoked = true; + return new ValueTask("custom_result"); }; + await using var session = await client.CreateSessionAsync(); await foreach (var msg in session.GetStreamingResponseAsync()) { @@ -268,7 +264,8 @@ public async Task GetStreamingResponseAsync_UnknownFunction_SendsErrorByDefault( }, }; - await using var session = new FunctionInvokingRealtimeClientSession(inner); + using var client = CreateClient(inner); + await using var session = await client.CreateSessionAsync(); await foreach (var msg in session.GetStreamingResponseAsync()) { @@ -305,10 +302,9 @@ public async Task GetStreamingResponseAsync_FunctionError_IncludesDetailedErrors }, }; - await using var session = new FunctionInvokingRealtimeClientSession(inner) - { - IncludeDetailedErrors = true, - }; + using var client = CreateClient(inner); + client.IncludeDetailedErrors = true; + await using var session = await client.CreateSessionAsync(); await foreach (var msg in session.GetStreamingResponseAsync()) { @@ -344,10 +340,9 @@ public async Task GetStreamingResponseAsync_FunctionError_HidesDetailsWhenNotEna }, }; - await using var session = new FunctionInvokingRealtimeClientSession(inner) - { - IncludeDetailedErrors = false, - }; + using var client = CreateClient(inner); + client.IncludeDetailedErrors = false; + await using var session = await client.CreateSessionAsync(); await foreach (var msg in session.GetStreamingResponseAsync()) { @@ -364,9 +359,10 @@ public async Task GetStreamingResponseAsync_FunctionError_HidesDetailsWhenNotEna public async Task GetService_ReturnsSelf() { await using var inner = new TestRealtimeClientSession(); - await using var session = new FunctionInvokingRealtimeClientSession(inner); + using var client = CreateClient(inner); + await using var session = await client.CreateSessionAsync(); - Assert.Same(session, session.GetService(typeof(FunctionInvokingRealtimeClientSession))); + Assert.Same(client, client.GetService(typeof(FunctionInvokingRealtimeClient))); Assert.Same(session, session.GetService(typeof(IRealtimeClientSession))); Assert.Same(inner, session.GetService(typeof(TestRealtimeClientSession))); } @@ -389,10 +385,9 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_StopsLoop() }, }; - await using var session = new FunctionInvokingRealtimeClientSession(inner) - { - TerminateOnUnknownCalls = true, - }; + using var client = CreateClient(inner); + client.TerminateOnUnknownCalls = true; + await using var session = await client.CreateSessionAsync(); var received = new List(); await foreach (var msg in session.GetStreamingResponseAsync()) @@ -424,10 +419,9 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_False_SendsE }, }; - await using var session = new FunctionInvokingRealtimeClientSession(inner) - { - TerminateOnUnknownCalls = false, - }; + using var client = CreateClient(inner); + client.TerminateOnUnknownCalls = false; + await using var session = await client.CreateSessionAsync(); var received = new List(); await foreach (var msg in session.GetStreamingResponseAsync()) @@ -502,10 +496,9 @@ public async Task GetStreamingResponseAsync_ConcurrentInvocation_InvokesInParall SendAsyncCallback = (_, _) => Task.CompletedTask, }; - await using var session = new FunctionInvokingRealtimeClientSession(inner) - { - AllowConcurrentInvocation = true, - }; + using var client = CreateClient(inner); + client.AllowConcurrentInvocation = true; + await using var session = await client.CreateSessionAsync(); await foreach (var msg in session.GetStreamingResponseAsync()) { @@ -545,10 +538,9 @@ public async Task GetStreamingResponseAsync_ConsecutiveErrors_ExceedsLimit_Throw SendAsyncCallback = (_, _) => Task.CompletedTask, }; - await using var session = new FunctionInvokingRealtimeClientSession(inner) - { - MaximumConsecutiveErrorsPerRequest = 1, - }; + using var client = CreateClient(inner); + client.MaximumConsecutiveErrorsPerRequest = 1; + await using var session = await client.CreateSessionAsync(); // Should eventually throw after exceeding the consecutive error limit await Assert.ThrowsAsync(async () => @@ -564,32 +556,34 @@ await Assert.ThrowsAsync(async () => public void UseFunctionInvocation_NullBuilder_Throws() { Assert.Throws("builder", () => - ((RealtimeClientSessionBuilder)null!).UseFunctionInvocation()); + ((RealtimeClientBuilder)null!).UseFunctionInvocation()); } [Fact] public async Task UseFunctionInvocation_ConfigureCallback_IsInvoked() { await using var inner = new TestRealtimeClientSession(); - var builder = new RealtimeClientSessionBuilder(inner); + using var innerClient = new TestRealtimeClient(inner); + var builder = new RealtimeClientBuilder(innerClient); bool configured = false; - builder.UseFunctionInvocation(configure: session => + builder.UseFunctionInvocation(configure: client => { configured = true; - session.IncludeDetailedErrors = true; - session.MaximumIterationsPerRequest = 10; + client.IncludeDetailedErrors = true; + client.MaximumIterationsPerRequest = 10; }); - await using var pipeline = builder.Build(); + using var pipeline = builder.Build(); Assert.True(configured); - var funcSession = pipeline.GetService(typeof(FunctionInvokingRealtimeClientSession)); - Assert.NotNull(funcSession); + await using var session = await pipeline.CreateSessionAsync(); - var typedSession = Assert.IsType(funcSession); - Assert.True(typedSession.IncludeDetailedErrors); - Assert.Equal(10, typedSession.MaximumIterationsPerRequest); + var funcClient = pipeline.GetService(typeof(FunctionInvokingRealtimeClient)); + Assert.NotNull(funcClient); + var typedClient = Assert.IsType(funcClient); + Assert.True(typedClient.IncludeDetailedErrors); + Assert.Equal(10, typedClient.MaximumIterationsPerRequest); } [Fact] @@ -615,7 +609,8 @@ public async Task GetStreamingResponseAsync_NonInvocableTool_TerminatesLoop() }, }; - await using var session = new FunctionInvokingRealtimeClientSession(inner); + using var client = CreateClient(inner); + await using var session = await client.CreateSessionAsync(); var received = new List(); await foreach (var msg in session.GetStreamingResponseAsync()) @@ -633,6 +628,13 @@ public async Task GetStreamingResponseAsync_NonInvocableTool_TerminatesLoop() #region Helpers +#pragma warning disable CA2000 // Dispose objects before losing scope - ownership transferred to FunctionInvokingRealtimeClient + private static FunctionInvokingRealtimeClient CreateClient(IRealtimeClientSession? session = null) + { + return new FunctionInvokingRealtimeClient(new TestRealtimeClient(session ?? new TestRealtimeClientSession())); + } +#pragma warning restore CA2000 + private static RealtimeServerResponseOutputItemMessage CreateFunctionCallOutputItemMessage( string callId, string functionName, IDictionary? arguments) { @@ -660,4 +662,24 @@ private static async IAsyncEnumerable YieldMessages( } #endregion + + private sealed class TestRealtimeClient : IRealtimeClient + { + private readonly IRealtimeClientSession _session; + + public TestRealtimeClient(IRealtimeClientSession session) + { + _session = session; + } + + public Task CreateSessionAsync(RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default) + => Task.FromResult(_session); + + public object? GetService(Type serviceType, object? serviceKey = null) => + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : _session.GetService(serviceType, serviceKey); + + public void Dispose() + { + } + } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeClientSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeClientTests.cs similarity index 73% rename from test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeClientSessionTests.cs rename to test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeClientTests.cs index c8c8d895c54..86f5e76956e 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeClientSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeClientTests.cs @@ -14,14 +14,15 @@ namespace Microsoft.Extensions.AI; -public class LoggingRealtimeClientSessionTests +public class LoggingRealtimeClientTests { [Fact] - public async Task LoggingRealtimeClientSession_InvalidArgs_Throws() + public async Task LoggingRealtimeClient_InvalidArgs_Throws() { await using var innerSession = new TestRealtimeClientSession(); - Assert.Throws("innerSession", () => new LoggingRealtimeClientSession(null!, NullLogger.Instance)); - Assert.Throws("logger", () => new LoggingRealtimeClientSession(innerSession, null!)); + using var innerClient = new TestRealtimeClient(innerSession); + Assert.Throws("innerClient", () => new LoggingRealtimeClient(null!, NullLogger.Instance)); + Assert.Throws("logger", () => new LoggingRealtimeClient(innerClient, null!)); } [Fact] @@ -29,18 +30,33 @@ public async Task UseLogging_AvoidsInjectingNopSession() { await using var innerSession = new TestRealtimeClientSession(); - Assert.Null(innerSession.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build().GetService(typeof(LoggingRealtimeClientSession))); - Assert.Same(innerSession, innerSession.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build().GetService(typeof(IRealtimeClientSession))); + using var c1 = new TestRealtimeClient(innerSession); + using var pipeline1 = c1.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build(); + await using var s1 = await pipeline1.CreateSessionAsync(); + Assert.Null(pipeline1.GetService(typeof(LoggingRealtimeClient))); + Assert.Same(innerSession, s1.GetService(typeof(IRealtimeClientSession))); using var factory = LoggerFactory.Create(b => b.AddFakeLogging()); - Assert.NotNull(innerSession.AsBuilder().UseLogging(factory).Build().GetService(typeof(LoggingRealtimeClientSession))); + using var c2 = new TestRealtimeClient(innerSession); + using var pipeline2 = c2.AsBuilder().UseLogging(factory).Build(); + await using var s2 = await pipeline2.CreateSessionAsync(); + Assert.NotNull(pipeline2.GetService(typeof(LoggingRealtimeClient))); ServiceCollection c = new(); c.AddFakeLogging(); var services = c.BuildServiceProvider(); - Assert.NotNull(innerSession.AsBuilder().UseLogging().Build(services).GetService(typeof(LoggingRealtimeClientSession))); - Assert.NotNull(innerSession.AsBuilder().UseLogging(null).Build(services).GetService(typeof(LoggingRealtimeClientSession))); - Assert.Null(innerSession.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build(services).GetService(typeof(LoggingRealtimeClientSession))); + using var c3 = new TestRealtimeClient(innerSession); + using var pipeline3 = c3.AsBuilder().UseLogging().Build(services); + await using var s3 = await pipeline3.CreateSessionAsync(); + Assert.NotNull(pipeline3.GetService(typeof(LoggingRealtimeClient))); + using var c4 = new TestRealtimeClient(innerSession); + using var pipeline4 = c4.AsBuilder().UseLogging(null).Build(services); + await using var s4 = await pipeline4.CreateSessionAsync(); + Assert.NotNull(pipeline4.GetService(typeof(LoggingRealtimeClient))); + using var c5 = new TestRealtimeClient(innerSession); + using var pipeline5 = c5.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build(services); + await using var s5 = await pipeline5.CreateSessionAsync(); + Assert.Null(pipeline5.GetService(typeof(LoggingRealtimeClient))); } [Theory] @@ -57,10 +73,12 @@ public async Task SendAsync_SessionUpdateMessage_LogsInvocationAndCompletion(Log await using var innerSession = new TestRealtimeClientSession(); - await using var session = innerSession + using var innerClient = new TestRealtimeClient(innerSession); + using var client = innerClient .AsBuilder() .UseLogging() .Build(services); + await using var session = await client.CreateSessionAsync(); await session.SendAsync(new RealtimeClientSessionUpdateMessage(new RealtimeSessionOptions { Model = "test-model", Instructions = "Be helpful" })); @@ -100,10 +118,12 @@ public async Task SendAsync_LogsInvocationAndCompletion(LogLevel level) SendAsyncCallback = (message, cancellationToken) => Task.CompletedTask, }; - await using var session = innerSession + using var innerClient = new TestRealtimeClient(innerSession); + using var client = innerClient .AsBuilder() .UseLogging() .Build(services); + await using var session = await client.CreateSessionAsync(); await session.SendAsync(new RealtimeClientMessage { MessageId = "test-event-123" }); @@ -147,10 +167,12 @@ static async IAsyncEnumerable GetMessagesAsync() yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.OutputAudioDelta, MessageId = "event-2" }; } - await using var session = innerSession + using var innerClient = new TestRealtimeClient(innerSession); + using var client = innerClient .AsBuilder() .UseLogging(loggerFactory) .Build(); + await using var session = await client.CreateSessionAsync(); await foreach (var message in session.GetStreamingResponseAsync()) { @@ -196,10 +218,12 @@ public async Task SendAsync_SessionUpdateMessage_LogsCancellation() }, }; - await using var session = innerSession + using var innerClient = new TestRealtimeClient(innerSession); + using var client = innerClient .AsBuilder() .UseLogging(loggerFactory) .Build(); + await using var session = await client.CreateSessionAsync(); cts.Cancel(); await Assert.ThrowsAsync(() => session.SendAsync(new RealtimeClientSessionUpdateMessage(new RealtimeSessionOptions()), cts.Token)); @@ -224,10 +248,12 @@ public async Task SendAsync_SessionUpdateMessage_LogsErrors() }, }; - await using var session = innerSession + using var innerClient = new TestRealtimeClient(innerSession); + using var client = innerClient .AsBuilder() .UseLogging(loggerFactory) .Build(); + await using var session = await client.CreateSessionAsync(); await Assert.ThrowsAsync(() => session.SendAsync(new RealtimeClientSessionUpdateMessage(new RealtimeSessionOptions()))); @@ -259,10 +285,12 @@ static async IAsyncEnumerable ThrowCancellationAsync([Enu #pragma warning restore CS0162 } - await using var session = innerSession + using var innerClient = new TestRealtimeClient(innerSession); + using var client = innerClient .AsBuilder() .UseLogging(loggerFactory) .Build(); + await using var session = await client.CreateSessionAsync(); cts.Cancel(); await Assert.ThrowsAsync(async () => @@ -299,10 +327,12 @@ static async IAsyncEnumerable ThrowErrorAsync() #pragma warning restore CS0162 } - await using var session = innerSession + using var innerClient = new TestRealtimeClient(innerSession); + using var client = innerClient .AsBuilder() .UseLogging(loggerFactory) .Build(); + await using var session = await client.CreateSessionAsync(); await Assert.ThrowsAsync(async () => { @@ -319,18 +349,20 @@ await Assert.ThrowsAsync(async () => } [Fact] - public async Task GetService_ReturnsLoggingSessionWhenRequested() + public async Task GetService_ReturnsLoggingClientWhenRequested() { using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddFakeLogging()); await using var innerSession = new TestRealtimeClientSession(); - await using var session = innerSession + using var innerClient = new TestRealtimeClient(innerSession); + using var client = innerClient .AsBuilder() .UseLogging(loggerFactory) .Build(); + await using var session = await client.CreateSessionAsync(); - Assert.NotNull(session.GetService(typeof(LoggingRealtimeClientSession))); + Assert.NotNull(client.GetService(typeof(LoggingRealtimeClient))); Assert.Same(session, session.GetService(typeof(IRealtimeClientSession))); } @@ -350,10 +382,12 @@ public async Task SendAsync_LogsCancellation() }, }; - await using var session = innerSession + using var innerClient = new TestRealtimeClient(innerSession); + using var client = innerClient .AsBuilder() .UseLogging(loggerFactory) .Build(); + await using var session = await client.CreateSessionAsync(); cts.Cancel(); await Assert.ThrowsAsync(() => @@ -379,10 +413,12 @@ public async Task SendAsync_LogsErrors() }, }; - await using var session = innerSession + using var innerClient = new TestRealtimeClient(innerSession); + using var client = innerClient .AsBuilder() .UseLogging(loggerFactory) .Build(); + await using var session = await client.CreateSessionAsync(); await Assert.ThrowsAsync(() => session.SendAsync(new RealtimeClientMessage())); @@ -397,46 +433,49 @@ await Assert.ThrowsAsync(() => public async Task JsonSerializerOptions_NullValue_Throws() { await using var innerSession = new TestRealtimeClientSession(); - await using var session = new LoggingRealtimeClientSession(innerSession, NullLogger.Instance); + using var innerClient = new TestRealtimeClient(innerSession); + using var client = new LoggingRealtimeClient(innerClient, NullLogger.Instance); - Assert.Throws("value", () => session.JsonSerializerOptions = null!); + Assert.Throws("value", () => client.JsonSerializerOptions = null!); } [Fact] public async Task JsonSerializerOptions_Roundtrip() { await using var innerSession = new TestRealtimeClientSession(); - await using var session = new LoggingRealtimeClientSession(innerSession, NullLogger.Instance); + using var innerClient = new TestRealtimeClient(innerSession); + using var client = new LoggingRealtimeClient(innerClient, NullLogger.Instance); var customOptions = new System.Text.Json.JsonSerializerOptions(); - session.JsonSerializerOptions = customOptions; + client.JsonSerializerOptions = customOptions; - Assert.Same(customOptions, session.JsonSerializerOptions); + Assert.Same(customOptions, client.JsonSerializerOptions); } [Fact] public void UseLogging_NullBuilder_Throws() { Assert.Throws("builder", () => - ((RealtimeClientSessionBuilder)null!).UseLogging()); + ((RealtimeClientBuilder)null!).UseLogging()); } - [Fact] - public async Task UseLogging_ConfigureCallback_IsInvoked() + private sealed class TestRealtimeClient : IRealtimeClient { - await using var innerSession = new TestRealtimeClientSession(); - using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddFakeLogging()); + private readonly IRealtimeClientSession _session; - bool configured = false; - await using var session = innerSession - .AsBuilder() - .UseLogging(loggerFactory, configure: s => - { - configured = true; - }) - .Build(); + public TestRealtimeClient(IRealtimeClientSession session) + { + _session = session; + } - Assert.True(configured); - } + public Task CreateSessionAsync(RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default) + => Task.FromResult(_session); + + public object? GetService(Type serviceType, object? serviceKey = null) => + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : _session.GetService(serviceType, serviceKey); + public void Dispose() + { + } + } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientTests.cs similarity index 85% rename from test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientSessionTests.cs rename to test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientTests.cs index a9d1fe76f6b..69865f5dec6 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientTests.cs @@ -14,7 +14,7 @@ namespace Microsoft.Extensions.AI; -public class OpenTelemetryRealtimeClientSessionTests +public class OpenTelemetryRealtimeClientTests { [Theory] [InlineData(false)] @@ -74,14 +74,11 @@ static async IAsyncEnumerable CallbackAsync([EnumeratorCa }; } - await using var session = innerSession - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName, configure: instance => - { - instance.EnableSensitiveData = enableSensitiveData; - instance.JsonSerializerOptions = TestJsonSerializerContext.Default.Options; - }) - .Build(); + using var innerClient = new TestRealtimeClient(innerSession); + using var client = new OpenTelemetryRealtimeClient(innerClient, sourceName: sourceName); + client.EnableSensitiveData = enableSensitiveData; + client.JsonSerializerOptions = TestJsonSerializerContext.Default.Options; + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesAsync()) { @@ -204,10 +201,12 @@ static async IAsyncEnumerable ThrowingCallbackAsync([Enum throw new InvalidOperationException("Streaming error"); } - await using var session = innerSession + using var innerClient = new TestRealtimeClient(innerSession); + using var client = innerClient .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); + await using var session = await client.CreateSessionAsync(); await Assert.ThrowsAsync(async () => { @@ -252,10 +251,12 @@ static async IAsyncEnumerable ErrorResponseCallbackAsync( }; } - await using var session = innerSession + using var innerClient = new TestRealtimeClient(innerSession); + using var client = innerClient .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesAsync()) { @@ -298,10 +299,12 @@ static async IAsyncEnumerable EmptyCallbackAsync([Enumera } var sourceName = Guid.NewGuid().ToString(); - await using var session = innerSession + using var innerClient = new TestRealtimeClient(innerSession); + using var client = innerClient .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); + await using var session = await client.CreateSessionAsync(); // This should work without errors even without listeners var count = 0; @@ -318,9 +321,11 @@ static async IAsyncEnumerable EmptyCallbackAsync([Enumera public async Task InvalidArgs_Throws() { await using var innerSession = new TestRealtimeClientSession(); + using var innerClient = new TestRealtimeClient(innerSession); - Assert.Throws("innerSession", () => new OpenTelemetryRealtimeClientSession(null!)); - Assert.Throws("value", () => new OpenTelemetryRealtimeClientSession(innerSession).JsonSerializerOptions = null!); + Assert.Throws("innerClient", () => new OpenTelemetryRealtimeClient(null!)); + using var client = new OpenTelemetryRealtimeClient(innerClient); + Assert.Throws("value", () => client.JsonSerializerOptions = null!); } [Fact] @@ -333,7 +338,9 @@ public void SessionUpdateMessage_NullOptions_Throws() public async Task GetService_ReturnsActivitySource() { await using var innerSession = new TestRealtimeClientSession(); - await using var session = new OpenTelemetryRealtimeClientSession(innerSession); + using var innerClient = new TestRealtimeClient(innerSession); + using var client = new OpenTelemetryRealtimeClient(innerClient); + await using var session = await client.CreateSessionAsync(); var activitySource = session.GetService(typeof(ActivitySource)); Assert.NotNull(activitySource); @@ -344,13 +351,13 @@ public async Task GetService_ReturnsActivitySource() public async Task GetService_ReturnsSelf() { await using var innerSession = new TestRealtimeClientSession(); - await using var session = new OpenTelemetryRealtimeClientSession(innerSession); + using var innerClient = new TestRealtimeClient(innerSession); + using var client = new OpenTelemetryRealtimeClient(innerClient); - var self = session.GetService(typeof(OpenTelemetryRealtimeClientSession)); - Assert.Same(session, self); + Assert.Same(client, client.GetService(typeof(OpenTelemetryRealtimeClient))); - var realtime = session.GetService(typeof(IRealtimeClientSession)); - Assert.Same(session, realtime); + await using var session = await client.CreateSessionAsync(); + Assert.Same(session, session.GetService(typeof(IRealtimeClientSession))); } [Fact] @@ -385,10 +392,12 @@ static async IAsyncEnumerable TranscriptionCallbackAsync( yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); } - await using var session = innerSession + using var innerClient = new TestRealtimeClient(innerSession); + using var client = innerClient .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesAsync()) { @@ -436,10 +445,12 @@ public async Task ToolChoiceMode_Logged(string modeKey, string expectedValue) GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - await using var session = innerSession + using var innerClient = new TestRealtimeClient(innerSession); + using var client = innerClient .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesAsync()) { @@ -475,10 +486,12 @@ public async Task AIFunction_ForcedTool_Logged() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - await using var session = innerSession + using var innerClient = new TestRealtimeClient(innerSession); + using var client = innerClient .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesAsync()) { @@ -515,10 +528,12 @@ public async Task RequireAny_ToolMode_Logged() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - await using var session = innerSession + using var innerClient = new TestRealtimeClient(innerSession); + using var client = innerClient .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesAsync()) { @@ -554,10 +569,12 @@ public async Task NoToolChoice_NotLogged() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - await using var session = innerSession + using var innerClient = new TestRealtimeClient(innerSession); + using var client = innerClient .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesAsync()) { @@ -591,10 +608,10 @@ public async Task ToolCallContentInClientMessages_LoggedAsInputMessages() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - await using var session = innerSession - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) - .Build(); + using var innerClient = new TestRealtimeClient(innerSession); + using var client = new OpenTelemetryRealtimeClient(innerClient, sourceName: sourceName); + client.EnableSensitiveData = true; + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesWithToolResultAsync()) { @@ -634,10 +651,10 @@ public async Task ToolCallContentInServerMessages_LoggedAsOutputMessages() GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithToolCallAsync(cancellationToken), }; - await using var session = innerSession - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) - .Build(); + using var innerClient = new TestRealtimeClient(innerSession); + using var client = new OpenTelemetryRealtimeClient(innerClient, sourceName: sourceName); + client.EnableSensitiveData = true; + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesAsync()) { @@ -677,10 +694,10 @@ public async Task ToolContentNotLoggedWithoutSensitiveData() GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithToolCallAsync(cancellationToken), }; - await using var session = innerSession - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = false) - .Build(); + using var innerClient = new TestRealtimeClient(innerSession); + using var client = new OpenTelemetryRealtimeClient(innerClient, sourceName: sourceName); + client.EnableSensitiveData = false; + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesWithToolResultAsync()) { @@ -762,10 +779,10 @@ public async Task AudioBufferAppendMessage_LoggedAsInputMessage() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - await using var session = innerSession - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) - .Build(); + using var innerClient = new TestRealtimeClient(innerSession); + using var client = new OpenTelemetryRealtimeClient(innerClient, sourceName: sourceName); + client.EnableSensitiveData = true; + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesAsync()) { @@ -805,10 +822,10 @@ public async Task AudioBufferCommitMessage_LoggedAsInputMessage() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - await using var session = innerSession - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) - .Build(); + using var innerClient = new TestRealtimeClient(innerSession); + using var client = new OpenTelemetryRealtimeClient(innerClient, sourceName: sourceName); + client.EnableSensitiveData = true; + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesAsync()) { @@ -848,10 +865,10 @@ public async Task ResponseCreateMessageWithInstructions_LoggedAsInputMessage() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - await using var session = innerSession - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) - .Build(); + using var innerClient = new TestRealtimeClient(innerSession); + using var client = new OpenTelemetryRealtimeClient(innerClient, sourceName: sourceName); + client.EnableSensitiveData = true; + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesWithInstructionsAsync()) { @@ -890,10 +907,10 @@ public async Task ResponseCreateMessageWithItems_LoggedAsInputMessage() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - await using var session = innerSession - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) - .Build(); + using var innerClient = new TestRealtimeClient(innerSession); + using var client = new OpenTelemetryRealtimeClient(innerClient, sourceName: sourceName); + client.EnableSensitiveData = true; + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesWithItemsAsync()) { @@ -932,10 +949,10 @@ public async Task OutputTextAudioMessage_LoggedAsOutputMessage() GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithTextOutputAsync(cancellationToken), }; - await using var session = innerSession - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) - .Build(); + using var innerClient = new TestRealtimeClient(innerSession); + using var client = new OpenTelemetryRealtimeClient(innerClient, sourceName: sourceName); + client.EnableSensitiveData = true; + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesAsync()) { @@ -973,10 +990,10 @@ public async Task InputAudioTranscriptionMessage_LoggedAsOutputMessage() GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithTranscriptionAsync(cancellationToken), }; - await using var session = innerSession - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) - .Build(); + using var innerClient = new TestRealtimeClient(innerSession); + using var client = new OpenTelemetryRealtimeClient(innerClient, sourceName: sourceName); + client.EnableSensitiveData = true; + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesAsync()) { @@ -1014,10 +1031,10 @@ public async Task ServerErrorMessage_LoggedAsOutputMessage() GetStreamingResponseAsyncCallback = (cancellationToken) => CallbackWithServerErrorAsync(cancellationToken), }; - await using var session = innerSession - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) - .Build(); + using var innerClient = new TestRealtimeClient(innerSession); + using var client = new OpenTelemetryRealtimeClient(innerClient, sourceName: sourceName); + client.EnableSensitiveData = true; + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesAsync()) { @@ -1055,10 +1072,10 @@ public async Task ConversationItemCreateWithTextContent_LoggedAsInputMessage() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - await using var session = innerSession - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) - .Build(); + using var innerClient = new TestRealtimeClient(innerSession); + using var client = new OpenTelemetryRealtimeClient(innerClient, sourceName: sourceName); + client.EnableSensitiveData = true; + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesWithTextContentAsync()) { @@ -1096,10 +1113,10 @@ public async Task DataContentInClientMessage_LoggedWithModality() GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), }; - await using var session = innerSession - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName, configure: s => s.EnableSensitiveData = true) - .Build(); + using var innerClient = new TestRealtimeClient(innerSession); + using var client = new OpenTelemetryRealtimeClient(innerClient, sourceName: sourceName); + client.EnableSensitiveData = true; + await using var session = await client.CreateSessionAsync(); await foreach (var msg in GetClientMessagesWithImageContentAsync()) { @@ -1200,37 +1217,36 @@ private static async IAsyncEnumerable CallbackWithServerE public void UseOpenTelemetry_NullBuilder_Throws() { Assert.Throws("builder", () => - ((RealtimeClientSessionBuilder)null!).UseOpenTelemetry()); + ((RealtimeClientBuilder)null!).UseOpenTelemetry()); } [Fact] - public async Task UseOpenTelemetry_ConfigureCallback_IsInvoked() + public async Task UseOpenTelemetry_BuildsPipeline() { await using var innerSession = new TestRealtimeClientSession(); - var builder = new RealtimeClientSessionBuilder(innerSession); + using var innerClient = new TestRealtimeClient(innerSession); + var builder = new RealtimeClientBuilder(innerClient); - bool configured = false; - builder.UseOpenTelemetry(configure: session => - { - configured = true; - session.EnableSensitiveData = true; - }); + builder.UseOpenTelemetry(); - await using var pipeline = builder.Build(); - Assert.True(configured); + using var pipeline = builder.Build(); + await using var session = await pipeline.CreateSessionAsync(); - var otelSession = pipeline.GetService(typeof(OpenTelemetryRealtimeClientSession)); - Assert.NotNull(otelSession); + var otelClient = pipeline.GetService(typeof(OpenTelemetryRealtimeClient)); + Assert.NotNull(otelClient); - var typedSession = Assert.IsType(otelSession); - Assert.True(typedSession.EnableSensitiveData); + var typedClient = Assert.IsType(otelClient); + typedClient.EnableSensitiveData = true; + Assert.True(typedClient.EnableSensitiveData); } [Fact] public async Task DisposeAsync_CanBeCalledMultipleTimes() { await using var innerSession = new TestRealtimeClientSession(); - var session = new OpenTelemetryRealtimeClientSession(innerSession); + using var innerClient = new TestRealtimeClient(innerSession); + using var client = new OpenTelemetryRealtimeClient(innerClient); + var session = await client.CreateSessionAsync(); await session.DisposeAsync(); await session.DisposeAsync(); @@ -1238,4 +1254,24 @@ public async Task DisposeAsync_CanBeCalledMultipleTimes() // Verifying no exception is thrown on double dispose Assert.NotNull(session); } + + private sealed class TestRealtimeClient : IRealtimeClient + { + private readonly IRealtimeClientSession _session; + + public TestRealtimeClient(IRealtimeClientSession session) + { + _session = session; + } + + public Task CreateSessionAsync(RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default) + => Task.FromResult(_session); + + public object? GetService(Type serviceType, object? serviceKey = null) => + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : _session.GetService(serviceType, serviceKey); + + public void Dispose() + { + } + } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientBuilderTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientBuilderTests.cs new file mode 100644 index 00000000000..5b4b7b8b7f4 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientBuilderTests.cs @@ -0,0 +1,182 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +#pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. + +namespace Microsoft.Extensions.AI; + +public class RealtimeClientBuilderTests +{ + [Fact] + public void Ctor_NullClient_Throws() + { + Assert.Throws("innerClient", () => new RealtimeClientBuilder((IRealtimeClient)null!)); + } + + [Fact] + public void Ctor_NullFactory_Throws() + { + Assert.Throws("innerClientFactory", () => new RealtimeClientBuilder((Func)null!)); + } + + [Fact] + public void Build_WithNoMiddleware_ReturnsInnerClient() + { + using var inner = new TestRealtimeClient(); + var builder = new RealtimeClientBuilder(inner); + + var result = builder.Build(); + Assert.Same(inner, result); + } + + [Fact] + public void Build_WithFactory_UsesFactory() + { + using var inner = new TestRealtimeClient(); + var builder = new RealtimeClientBuilder(_ => inner); + + var result = builder.Build(); + Assert.Same(inner, result); + } + + [Fact] + public void Use_NullClientFactory_Throws() + { + using var inner = new TestRealtimeClient(); + var builder = new RealtimeClientBuilder(inner); + + Assert.Throws("clientFactory", () => builder.Use((Func)null!)); + Assert.Throws("clientFactory", () => builder.Use((Func)null!)); + } + + [Fact] + public void Build_PipelineOrder_FirstAddedIsOutermost() + { + var callOrder = new List(); + using var inner = new TestRealtimeClient(); + + var builder = new RealtimeClientBuilder(inner); + builder.Use(client => new OrderTrackingClient(client, "first", callOrder)); + builder.Use(client => new OrderTrackingClient(client, "second", callOrder)); + + using var pipeline = builder.Build(); + + // The outermost should be "first" (added first) + var outermost = Assert.IsType(pipeline); + Assert.Equal("first", outermost.Name); + + var middle = Assert.IsType(outermost.GetInner()); + Assert.Equal("second", middle.Name); + + Assert.Same(inner, middle.GetInner()); + } + + [Fact] + public void Build_WithServiceProvider_PassesToFactory() + { + IServiceProvider? capturedServices = null; + using var inner = new TestRealtimeClient(); + + var builder = new RealtimeClientBuilder(inner); + builder.Use((client, services) => + { + capturedServices = services; + return client; + }); + + var services = new EmptyServiceProvider(); + builder.Build(services); + + Assert.Same(services, capturedServices); + } + + [Fact] + public void Build_NullServiceProvider_UsesEmptyProvider() + { + IServiceProvider? capturedServices = null; + using var inner = new TestRealtimeClient(); + + var builder = new RealtimeClientBuilder(inner); + builder.Use((client, services) => + { + capturedServices = services; + return client; + }); + + builder.Build(null); + + Assert.NotNull(capturedServices); + } + + [Fact] + public void Use_ReturnsSameBuilder_ForChaining() + { + using var inner = new TestRealtimeClient(); + var builder = new RealtimeClientBuilder(inner); + + var returned = builder.Use(c => c); + Assert.Same(builder, returned); + } + + [Fact] + public void AsBuilder_NullClient_Throws() + { + Assert.Throws("innerClient", () => ((IRealtimeClient)null!).AsBuilder()); + } + + [Fact] + public void AsBuilder_ReturnsBuilder() + { + using var inner = new TestRealtimeClient(); + var builder = inner.AsBuilder(); + + Assert.NotNull(builder); + Assert.Same(inner, builder.Build()); + } + + private sealed class TestRealtimeClient : IRealtimeClient + { + public Task CreateSessionAsync(RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default) + => Task.FromResult(new TestRealtimeClientSession()); + + public object? GetService(Type serviceType, object? serviceKey = null) => + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : null; + + public void Dispose() + { + } + } + + private sealed class OrderTrackingClient : DelegatingRealtimeClient + { + public string Name { get; } + private readonly List _callOrder; + + public OrderTrackingClient(IRealtimeClient inner, string name, List callOrder) + : base(inner) + { + Name = name; + _callOrder = callOrder; + } + + public IRealtimeClient GetInner() => InnerClient; + + public override async Task CreateSessionAsync( + RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default) + { + _callOrder.Add(Name); + return await base.CreateSessionAsync(options, cancellationToken); + } + } + + private sealed class EmptyServiceProvider : IServiceProvider + { + public object? GetService(Type serviceType) => null; + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientSessionBuilderTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientSessionBuilderTests.cs deleted file mode 100644 index 0ba2ae72d32..00000000000 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientSessionBuilderTests.cs +++ /dev/null @@ -1,173 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; -using Xunit; - -#pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. - -namespace Microsoft.Extensions.AI; - -public class RealtimeClientSessionBuilderTests -{ - [Fact] - public void Ctor_NullSession_Throws() - { - Assert.Throws("innerSession", () => new RealtimeClientSessionBuilder((IRealtimeClientSession)null!)); - } - - [Fact] - public void Ctor_NullFactory_Throws() - { - Assert.Throws("innerSessionFactory", () => new RealtimeClientSessionBuilder((Func)null!)); - } - - [Fact] - public async Task Build_WithNoMiddleware_ReturnsInnerSession() - { - await using var inner = new TestRealtimeClientSession(); - var builder = new RealtimeClientSessionBuilder(inner); - - var result = builder.Build(); - Assert.Same(inner, result); - } - - [Fact] - public async Task Build_WithFactory_UsesFactory() - { - await using var inner = new TestRealtimeClientSession(); - var builder = new RealtimeClientSessionBuilder(_ => inner); - - var result = builder.Build(); - Assert.Same(inner, result); - } - - [Fact] - public async Task Use_NullSessionFactory_Throws() - { - await using var inner = new TestRealtimeClientSession(); - var builder = new RealtimeClientSessionBuilder(inner); - - Assert.Throws("sessionFactory", () => builder.Use((Func)null!)); - Assert.Throws("sessionFactory", () => builder.Use((Func)null!)); - } - - [Fact] - public async Task Build_PipelineOrder_FirstAddedIsOutermost() - { - var callOrder = new List(); - await using var inner = new TestRealtimeClientSession(); - - var builder = new RealtimeClientSessionBuilder(inner); - builder.Use(session => new OrderTrackingClientSession(session, "first", callOrder)); - builder.Use(session => new OrderTrackingClientSession(session, "second", callOrder)); - - await using var pipeline = builder.Build(); - - // The outermost should be "first" (added first) - var outermost = Assert.IsType(pipeline); - Assert.Equal("first", outermost.Name); - - var middle = Assert.IsType(outermost.GetInner()); - Assert.Equal("second", middle.Name); - - Assert.Same(inner, middle.GetInner()); - } - - [Fact] - public async Task Build_WithServiceProvider_PassesToFactory() - { - IServiceProvider? capturedServices = null; - await using var inner = new TestRealtimeClientSession(); - - var builder = new RealtimeClientSessionBuilder(inner); - builder.Use((session, services) => - { - capturedServices = services; - return session; - }); - - var services = new EmptyServiceProvider(); - builder.Build(services); - - Assert.Same(services, capturedServices); - } - - [Fact] - public async Task Build_NullServiceProvider_UsesEmptyProvider() - { - IServiceProvider? capturedServices = null; - await using var inner = new TestRealtimeClientSession(); - - var builder = new RealtimeClientSessionBuilder(inner); - builder.Use((session, services) => - { - capturedServices = services; - return session; - }); - - builder.Build(null); - - Assert.NotNull(capturedServices); - } - - [Fact] - public async Task Use_ReturnsSameBuilder_ForChaining() - { - await using var inner = new TestRealtimeClientSession(); - var builder = new RealtimeClientSessionBuilder(inner); - - var returned = builder.Use(s => s); - Assert.Same(builder, returned); - } - - [Fact] - public void AsBuilder_NullSession_Throws() - { - Assert.Throws("innerSession", () => ((IRealtimeClientSession)null!).AsBuilder()); - } - - [Fact] - public async Task AsBuilder_ReturnsBuilder() - { - await using var inner = new TestRealtimeClientSession(); - var builder = inner.AsBuilder(); - - Assert.NotNull(builder); - Assert.Same(inner, builder.Build()); - } - - private sealed class OrderTrackingClientSession : DelegatingRealtimeClientSession - { - public string Name { get; } - private readonly List _callOrder; - - public OrderTrackingClientSession(IRealtimeClientSession inner, string name, List callOrder) - : base(inner) - { - Name = name; - _callOrder = callOrder; - } - - public IRealtimeClientSession GetInner() => InnerSession; - - public override async IAsyncEnumerable GetStreamingResponseAsync( - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - _callOrder.Add(Name); - await foreach (var msg in base.GetStreamingResponseAsync(cancellationToken).ConfigureAwait(false)) - { - yield return msg; - } - } - } - - private sealed class EmptyServiceProvider : IServiceProvider - { - public object? GetService(Type serviceType) => null; - } -} From 39ad4b009bbb1a2075395fb97b592a791f5d5544 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Fri, 6 Mar 2026 10:10:22 -0800 Subject: [PATCH 69/92] Rename message classes: move RealtimeClient/RealtimeServer before Message Rename pattern: RealtimeClient{Name}Message -> {Name}RealtimeClientMessage Rename pattern: RealtimeServer{Name}Message -> {Name}RealtimeServerMessage Base classes RealtimeClientMessage and RealtimeServerMessage unchanged. --- ...eConversationItemRealtimeClientMessage.cs} | 6 +- ...=> CreateResponseRealtimeClientMessage.cs} | 6 +- ...ssage.cs => ErrorRealtimeServerMessage.cs} | 6 +- ...AudioBufferAppendRealtimeClientMessage.cs} | 6 +- ...AudioBufferCommitRealtimeClientMessage.cs} | 6 +- ...udioTranscriptionRealtimeServerMessage.cs} | 6 +- ...> OutputTextAudioRealtimeServerMessage.cs} | 6 +- .../Realtime/RealtimeSessionOptions.cs | 2 +- ...> ResponseCreatedRealtimeServerMessage.cs} | 6 +- ...esponseOutputItemRealtimeServerMessage.cs} | 6 +- ... => SessionUpdateRealtimeClientMessage.cs} | 6 +- .../OpenAIRealtimeClient.cs | 2 +- .../OpenAIRealtimeClientSession.cs | 62 +++++++++---------- .../FunctionInvokingRealtimeClient.cs | 4 +- .../FunctionInvokingRealtimeClientSession.cs | 14 ++--- .../OpenTelemetryRealtimeClientSession.cs | 26 ++++---- .../Realtime/RealtimeClientMessageTests.cs | 22 +++---- .../Realtime/RealtimeServerMessageTests.cs | 30 ++++----- .../OpenAIRealtimeClientSessionTests.cs | 2 +- .../FunctionInvokingRealtimeClientTests.cs | 20 +++--- .../Realtime/LoggingRealtimeClientTests.cs | 6 +- .../OpenTelemetryRealtimeClientTests.cs | 58 ++++++++--------- 22 files changed, 154 insertions(+), 154 deletions(-) rename src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/{RealtimeClientCreateConversationItemMessage.cs => CreateConversationItemRealtimeClientMessage.cs} (83%) rename src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/{RealtimeClientCreateResponseMessage.cs => CreateResponseRealtimeClientMessage.cs} (95%) rename src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/{RealtimeServerErrorMessage.cs => ErrorRealtimeServerMessage.cs} (85%) rename src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/{RealtimeClientInputAudioBufferAppendMessage.cs => InputAudioBufferAppendRealtimeClientMessage.cs} (82%) rename src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/{RealtimeClientInputAudioBufferCommitMessage.cs => InputAudioBufferCommitRealtimeClientMessage.cs} (69%) rename src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/{RealtimeServerInputAudioTranscriptionMessage.cs => InputAudioTranscriptionRealtimeServerMessage.cs} (89%) rename src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/{RealtimeServerOutputTextAudioMessage.cs => OutputTextAudioRealtimeServerMessage.cs} (91%) rename src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/{RealtimeServerResponseCreatedMessage.cs => ResponseCreatedRealtimeServerMessage.cs} (94%) rename src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/{RealtimeServerResponseOutputItemMessage.cs => ResponseOutputItemRealtimeServerMessage.cs} (88%) rename src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/{RealtimeClientSessionUpdateMessage.cs => SessionUpdateRealtimeClientMessage.cs} (85%) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateConversationItemMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateConversationItemRealtimeClientMessage.cs similarity index 83% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateConversationItemMessage.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateConversationItemRealtimeClientMessage.cs index 44edff595f7..20c6288ac0a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateConversationItemMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateConversationItemRealtimeClientMessage.cs @@ -11,14 +11,14 @@ namespace Microsoft.Extensions.AI; /// Represents a real-time message for creating a conversation item. /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class RealtimeClientCreateConversationItemMessage : RealtimeClientMessage +public class CreateConversationItemRealtimeClientMessage : RealtimeClientMessage { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The conversation item to create. /// The optional ID of the previous conversation item to insert the new one after. - public RealtimeClientCreateConversationItemMessage(RealtimeConversationItem item, string? previousId = null) + public CreateConversationItemRealtimeClientMessage(RealtimeConversationItem item, string? previousId = null) { PreviousId = previousId; Item = Throw.IfNull(item); diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateResponseMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateResponseRealtimeClientMessage.cs similarity index 95% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateResponseMessage.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateResponseRealtimeClientMessage.cs index bdfe74ac316..0f4498a8f83 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientCreateResponseMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateResponseRealtimeClientMessage.cs @@ -17,12 +17,12 @@ namespace Microsoft.Extensions.AI; /// for this response only. /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class RealtimeClientCreateResponseMessage : RealtimeClientMessage +public class CreateResponseRealtimeClientMessage : RealtimeClientMessage { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - public RealtimeClientCreateResponseMessage() + public CreateResponseRealtimeClientMessage() { } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ErrorRealtimeServerMessage.cs similarity index 85% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ErrorRealtimeServerMessage.cs index c8a9eac227d..8a606f53a82 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerErrorMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ErrorRealtimeServerMessage.cs @@ -13,12 +13,12 @@ namespace Microsoft.Extensions.AI; /// Used with the . /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class RealtimeServerErrorMessage : RealtimeServerMessage +public class ErrorRealtimeServerMessage : RealtimeServerMessage { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - public RealtimeServerErrorMessage() + public ErrorRealtimeServerMessage() { Type = RealtimeServerMessageType.Error; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/InputAudioBufferAppendRealtimeClientMessage.cs similarity index 82% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/InputAudioBufferAppendRealtimeClientMessage.cs index 1fc9724c5ee..1f20903fb74 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferAppendMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/InputAudioBufferAppendRealtimeClientMessage.cs @@ -11,15 +11,15 @@ namespace Microsoft.Extensions.AI; /// Represents a real-time message for appending audio buffer input. /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class RealtimeClientInputAudioBufferAppendMessage : RealtimeClientMessage +public class InputAudioBufferAppendRealtimeClientMessage : RealtimeClientMessage { private DataContent _content; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The data content containing the audio buffer data to append. - public RealtimeClientInputAudioBufferAppendMessage(DataContent audioContent) + public InputAudioBufferAppendRealtimeClientMessage(DataContent audioContent) { _content = Throw.IfNull(audioContent); } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/InputAudioBufferCommitRealtimeClientMessage.cs similarity index 69% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/InputAudioBufferCommitRealtimeClientMessage.cs index 8e7c61a7abc..427fbda5ca9 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientInputAudioBufferCommitMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/InputAudioBufferCommitRealtimeClientMessage.cs @@ -10,12 +10,12 @@ namespace Microsoft.Extensions.AI; /// Represents a real-time message for committing audio buffer input. /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class RealtimeClientInputAudioBufferCommitMessage : RealtimeClientMessage +public class InputAudioBufferCommitRealtimeClientMessage : RealtimeClientMessage { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - public RealtimeClientInputAudioBufferCommitMessage() + public InputAudioBufferCommitRealtimeClientMessage() { } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/InputAudioTranscriptionRealtimeServerMessage.cs similarity index 89% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/InputAudioTranscriptionRealtimeServerMessage.cs index ec011d3faee..c50d7c8240f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerInputAudioTranscriptionMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/InputAudioTranscriptionRealtimeServerMessage.cs @@ -13,16 +13,16 @@ namespace Microsoft.Extensions.AI; /// Used when having InputAudioTranscriptionCompleted, InputAudioTranscriptionDelta, or InputAudioTranscriptionFailed response types. /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class RealtimeServerInputAudioTranscriptionMessage : RealtimeServerMessage +public class InputAudioTranscriptionRealtimeServerMessage : RealtimeServerMessage { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The type of the real-time server response. /// /// The parameter should be InputAudioTranscriptionCompleted, InputAudioTranscriptionDelta, or InputAudioTranscriptionFailed. /// - public RealtimeServerInputAudioTranscriptionMessage(RealtimeServerMessageType type) + public InputAudioTranscriptionRealtimeServerMessage(RealtimeServerMessageType type) { Type = type; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/OutputTextAudioRealtimeServerMessage.cs similarity index 91% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/OutputTextAudioRealtimeServerMessage.cs index be6b9137bfc..60eab81aedf 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerOutputTextAudioMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/OutputTextAudioRealtimeServerMessage.cs @@ -10,10 +10,10 @@ namespace Microsoft.Extensions.AI; /// Represents a real-time server message for output text and audio. /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class RealtimeServerOutputTextAudioMessage : RealtimeServerMessage +public class OutputTextAudioRealtimeServerMessage : RealtimeServerMessage { /// - /// Initializes a new instance of the class for handling output text delta responses. + /// Initializes a new instance of the class for handling output text delta responses. /// /// The type of the real-time server response. /// @@ -21,7 +21,7 @@ public class RealtimeServerOutputTextAudioMessage : RealtimeServerMessage /// , , /// , or . /// - public RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType type) + public OutputTextAudioRealtimeServerMessage(RealtimeServerMessageType type) { Type = type; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs index 4f1ad9c9242..0866324e664 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs @@ -77,7 +77,7 @@ public class RealtimeSessionOptions /// /// /// The underlying implementation might have its own representation of options. - /// When a is sent with a , + /// When a is sent with a , /// that implementation might convert the provided options into its own representation in order to use it while /// performing the operation. For situations where a consumer knows which concrete /// is being used and how it represents options, a new instance of that implementation-specific options type can be diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ResponseCreatedRealtimeServerMessage.cs similarity index 94% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ResponseCreatedRealtimeServerMessage.cs index a6934d45f32..d40f0ee0433 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseCreatedMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ResponseCreatedRealtimeServerMessage.cs @@ -22,15 +22,15 @@ namespace Microsoft.Extensions.AI; /// /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class RealtimeServerResponseCreatedMessage : RealtimeServerMessage +public class ResponseCreatedRealtimeServerMessage : RealtimeServerMessage { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// /// The should be or . /// - public RealtimeServerResponseCreatedMessage(RealtimeServerMessageType type) + public ResponseCreatedRealtimeServerMessage(RealtimeServerMessageType type) { Type = type; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ResponseOutputItemRealtimeServerMessage.cs similarity index 88% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ResponseOutputItemRealtimeServerMessage.cs index 38e9143b075..24ee1dfc2af 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerResponseOutputItemMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ResponseOutputItemRealtimeServerMessage.cs @@ -21,15 +21,15 @@ namespace Microsoft.Extensions.AI; /// /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class RealtimeServerResponseOutputItemMessage : RealtimeServerMessage +public class ResponseOutputItemRealtimeServerMessage : RealtimeServerMessage { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// /// The should be or . /// - public RealtimeServerResponseOutputItemMessage(RealtimeServerMessageType type) + public ResponseOutputItemRealtimeServerMessage(RealtimeServerMessageType type) { Type = type; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientSessionUpdateMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SessionUpdateRealtimeClientMessage.cs similarity index 85% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientSessionUpdateMessage.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SessionUpdateRealtimeClientMessage.cs index 6d0f75def20..a2c1bbb614f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeClientSessionUpdateMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SessionUpdateRealtimeClientMessage.cs @@ -22,13 +22,13 @@ namespace Microsoft.Extensions.AI; /// /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] -public class RealtimeClientSessionUpdateMessage : RealtimeClientMessage +public class SessionUpdateRealtimeClientMessage : RealtimeClientMessage { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The session options to apply. - public RealtimeClientSessionUpdateMessage(RealtimeSessionOptions options) + public SessionUpdateRealtimeClientMessage(RealtimeSessionOptions options) { Options = Throw.IfNull(options); } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs index 05dfc20b469..40689b71cfc 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClient.cs @@ -63,7 +63,7 @@ public async Task CreateSessionAsync(RealtimeSessionOpti { if (options is not null) { - await session.SendAsync(new RealtimeClientSessionUpdateMessage(options), cancellationToken).ConfigureAwait(false); + await session.SendAsync(new SessionUpdateRealtimeClientMessage(options), cancellationToken).ConfigureAwait(false); } return session; diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs index 1feff1e2316..f0c4bd85b6e 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs @@ -120,23 +120,23 @@ public async Task SendAsync(RealtimeClientMessage message, CancellationToken can { switch (message) { - case RealtimeClientSessionUpdateMessage sessionUpdate: + case SessionUpdateRealtimeClientMessage sessionUpdate: await UpdateSessionAsync(sessionUpdate.Options, cancellationToken).ConfigureAwait(false); break; - case RealtimeClientCreateResponseMessage responseCreate: + case CreateResponseRealtimeClientMessage responseCreate: await SendResponseCreateAsync(responseCreate, cancellationToken).ConfigureAwait(false); break; - case RealtimeClientCreateConversationItemMessage itemCreate: + case CreateConversationItemRealtimeClientMessage itemCreate: await SendConversationItemCreateAsync(itemCreate, cancellationToken).ConfigureAwait(false); break; - case RealtimeClientInputAudioBufferAppendMessage audioAppend: + case InputAudioBufferAppendRealtimeClientMessage audioAppend: await SendInputAudioAppendAsync(audioAppend, cancellationToken).ConfigureAwait(false); break; - case RealtimeClientInputAudioBufferCommitMessage: + case InputAudioBufferCommitRealtimeClientMessage: if (message.MessageId is not null) { var cmd = new Sdk.RealtimeClientCommandInputAudioBufferCommit { EventId = message.MessageId }; @@ -206,7 +206,7 @@ public ValueTask DisposeAsync() #region Send Helpers (MEAI → SDK) - private async Task SendResponseCreateAsync(RealtimeClientCreateResponseMessage responseCreate, CancellationToken cancellationToken) + private async Task SendResponseCreateAsync(CreateResponseRealtimeClientMessage responseCreate, CancellationToken cancellationToken) { var responseOptions = new Sdk.RealtimeResponseOptions(); @@ -305,7 +305,7 @@ private async Task SendResponseCreateAsync(RealtimeClientCreateResponseMessage r } } - private async Task SendConversationItemCreateAsync(RealtimeClientCreateConversationItemMessage itemCreate, CancellationToken cancellationToken) + private async Task SendConversationItemCreateAsync(CreateConversationItemRealtimeClientMessage itemCreate, CancellationToken cancellationToken) { if (itemCreate.Item is null) { @@ -333,7 +333,7 @@ private async Task SendConversationItemCreateAsync(RealtimeClientCreateConversat } } - private async Task SendInputAudioAppendAsync(RealtimeClientInputAudioBufferAppendMessage audioAppend, CancellationToken cancellationToken) + private async Task SendInputAudioAppendAsync(InputAudioBufferAppendRealtimeClientMessage audioAppend, CancellationToken cancellationToken) { if (audioAppend.Content is null || !audioAppend.Content.HasTopLevelMediaType("audio")) { @@ -718,7 +718,7 @@ private static BinaryData ExtractAudioBinaryData(DataContent content) Sdk.RealtimeServerUpdateResponseDone e => MapResponseCreatedOrDone(e.EventId, e.Response, RealtimeServerMessageType.ResponseDone, e), Sdk.RealtimeServerUpdateResponseOutputItemAdded e => MapResponseOutputItem(e.EventId, e.ResponseId, e.OutputIndex, e.Item, RealtimeServerMessageType.ResponseOutputItemAdded, e), Sdk.RealtimeServerUpdateResponseOutputItemDone e => MapResponseOutputItem(e.EventId, e.ResponseId, e.OutputIndex, e.Item, RealtimeServerMessageType.ResponseOutputItemDone, e), - Sdk.RealtimeServerUpdateResponseOutputAudioDelta e => new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputAudioDelta) + Sdk.RealtimeServerUpdateResponseOutputAudioDelta e => new OutputTextAudioRealtimeServerMessage(RealtimeServerMessageType.OutputAudioDelta) { MessageId = e.EventId, ResponseId = e.ResponseId, @@ -728,7 +728,7 @@ private static BinaryData ExtractAudioBinaryData(DataContent content) Audio = e.Delta is not null ? Convert.ToBase64String(e.Delta.ToArray()) : null, RawRepresentation = e, }, - Sdk.RealtimeServerUpdateResponseOutputAudioDone e => new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputAudioDone) + Sdk.RealtimeServerUpdateResponseOutputAudioDone e => new OutputTextAudioRealtimeServerMessage(RealtimeServerMessageType.OutputAudioDone) { MessageId = e.EventId, ResponseId = e.ResponseId, @@ -737,7 +737,7 @@ private static BinaryData ExtractAudioBinaryData(DataContent content) ContentIndex = e.ContentIndex, RawRepresentation = e, }, - Sdk.RealtimeServerUpdateResponseOutputAudioTranscriptDelta e => new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputAudioTranscriptionDelta) + Sdk.RealtimeServerUpdateResponseOutputAudioTranscriptDelta e => new OutputTextAudioRealtimeServerMessage(RealtimeServerMessageType.OutputAudioTranscriptionDelta) { MessageId = e.EventId, ResponseId = e.ResponseId, @@ -747,7 +747,7 @@ private static BinaryData ExtractAudioBinaryData(DataContent content) Text = e.Delta, RawRepresentation = e, }, - Sdk.RealtimeServerUpdateResponseOutputAudioTranscriptDone e => new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputAudioTranscriptionDone) + Sdk.RealtimeServerUpdateResponseOutputAudioTranscriptDone e => new OutputTextAudioRealtimeServerMessage(RealtimeServerMessageType.OutputAudioTranscriptionDone) { MessageId = e.EventId, ResponseId = e.ResponseId, @@ -775,9 +775,9 @@ private static BinaryData ExtractAudioBinaryData(DataContent content) }, }; - private static RealtimeServerErrorMessage MapError(Sdk.RealtimeServerUpdateError e) + private static ErrorRealtimeServerMessage MapError(Sdk.RealtimeServerUpdateError e) { - var msg = new RealtimeServerErrorMessage + var msg = new ErrorRealtimeServerMessage { MessageId = e.EventId, Error = new ErrorContent(e.Error?.Message), @@ -876,10 +876,10 @@ private RealtimeSessionOptions MapConversationSessionToOptions(Sdk.RealtimeConve }; } - private static RealtimeServerResponseCreatedMessage MapResponseCreatedOrDone( + private static ResponseCreatedRealtimeServerMessage MapResponseCreatedOrDone( string? eventId, Sdk.RealtimeResponse? response, RealtimeServerMessageType type, Sdk.RealtimeServerUpdate update) { - var msg = new RealtimeServerResponseCreatedMessage(type) + var msg = new ResponseCreatedRealtimeServerMessage(type) { MessageId = eventId, RawRepresentation = update, @@ -953,11 +953,11 @@ private static RealtimeServerResponseCreatedMessage MapResponseCreatedOrDone( return msg; } - private static RealtimeServerResponseOutputItemMessage MapResponseOutputItem( + private static ResponseOutputItemRealtimeServerMessage MapResponseOutputItem( string? eventId, string? responseId, int outputIndex, Sdk.RealtimeItem? item, RealtimeServerMessageType type, Sdk.RealtimeServerUpdate update) { - return new RealtimeServerResponseOutputItemMessage(type) + return new ResponseOutputItemRealtimeServerMessage(type) { MessageId = eventId, ResponseId = responseId, @@ -967,20 +967,20 @@ private static RealtimeServerResponseOutputItemMessage MapResponseOutputItem( }; } - private static RealtimeServerResponseOutputItemMessage MapConversationItem( + private static ResponseOutputItemRealtimeServerMessage MapConversationItem( string? eventId, Sdk.RealtimeItem? item, RealtimeServerMessageType type, Sdk.RealtimeServerUpdate update) { var mapped = item is not null ? MapRealtimeItem(item) : null; if (mapped is null) { - return new RealtimeServerResponseOutputItemMessage(RealtimeServerMessageType.RawContentOnly) + return new ResponseOutputItemRealtimeServerMessage(RealtimeServerMessageType.RawContentOnly) { MessageId = eventId, RawRepresentation = update, }; } - return new RealtimeServerResponseOutputItemMessage(type) + return new ResponseOutputItemRealtimeServerMessage(type) { MessageId = eventId, Item = mapped, @@ -988,9 +988,9 @@ private static RealtimeServerResponseOutputItemMessage MapConversationItem( }; } - private static RealtimeServerInputAudioTranscriptionMessage MapInputTranscriptionDelta(Sdk.RealtimeServerUpdateConversationItemInputAudioTranscriptionDelta e) + private static InputAudioTranscriptionRealtimeServerMessage MapInputTranscriptionDelta(Sdk.RealtimeServerUpdateConversationItemInputAudioTranscriptionDelta e) { - return new RealtimeServerInputAudioTranscriptionMessage(RealtimeServerMessageType.InputAudioTranscriptionDelta) + return new InputAudioTranscriptionRealtimeServerMessage(RealtimeServerMessageType.InputAudioTranscriptionDelta) { MessageId = e.EventId, ItemId = e.ItemId, @@ -1000,9 +1000,9 @@ private static RealtimeServerInputAudioTranscriptionMessage MapInputTranscriptio }; } - private static RealtimeServerInputAudioTranscriptionMessage MapInputTranscriptionCompleted(Sdk.RealtimeServerUpdateConversationItemInputAudioTranscriptionCompleted e) + private static InputAudioTranscriptionRealtimeServerMessage MapInputTranscriptionCompleted(Sdk.RealtimeServerUpdateConversationItemInputAudioTranscriptionCompleted e) { - return new RealtimeServerInputAudioTranscriptionMessage(RealtimeServerMessageType.InputAudioTranscriptionCompleted) + return new InputAudioTranscriptionRealtimeServerMessage(RealtimeServerMessageType.InputAudioTranscriptionCompleted) { MessageId = e.EventId, ItemId = e.ItemId, @@ -1012,9 +1012,9 @@ private static RealtimeServerInputAudioTranscriptionMessage MapInputTranscriptio }; } - private static RealtimeServerInputAudioTranscriptionMessage MapInputTranscriptionFailed(Sdk.RealtimeServerUpdateConversationItemInputAudioTranscriptionFailed e) + private static InputAudioTranscriptionRealtimeServerMessage MapInputTranscriptionFailed(Sdk.RealtimeServerUpdateConversationItemInputAudioTranscriptionFailed e) { - var msg = new RealtimeServerInputAudioTranscriptionMessage(RealtimeServerMessageType.InputAudioTranscriptionFailed) + var msg = new InputAudioTranscriptionRealtimeServerMessage(RealtimeServerMessageType.InputAudioTranscriptionFailed) { MessageId = e.EventId, ItemId = e.ItemId, @@ -1034,10 +1034,10 @@ private static RealtimeServerInputAudioTranscriptionMessage MapInputTranscriptio return msg; } - private static RealtimeServerResponseOutputItemMessage MapMcpCallEvent( + private static ResponseOutputItemRealtimeServerMessage MapMcpCallEvent( string? eventId, string? itemId, int outputIndex, RealtimeServerMessageType type, Sdk.RealtimeServerUpdate update) { - return new RealtimeServerResponseOutputItemMessage(type) + return new ResponseOutputItemRealtimeServerMessage(type) { MessageId = eventId, Item = itemId is not null ? new RealtimeConversationItem([], itemId) : null, @@ -1046,10 +1046,10 @@ private static RealtimeServerResponseOutputItemMessage MapMcpCallEvent( }; } - private static RealtimeServerResponseOutputItemMessage MapMcpListToolsEvent( + private static ResponseOutputItemRealtimeServerMessage MapMcpListToolsEvent( string? eventId, string? itemId, RealtimeServerMessageType type, Sdk.RealtimeServerUpdate update) { - return new RealtimeServerResponseOutputItemMessage(type) + return new ResponseOutputItemRealtimeServerMessage(type) { MessageId = eventId, Item = itemId is not null ? new RealtimeConversationItem([], itemId) : null, diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClient.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClient.cs index ce1627086e5..0510965c8be 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClient.cs @@ -13,14 +13,14 @@ namespace Microsoft.Extensions.AI; /// -/// A delegating realtime client that invokes functions defined on . +/// A delegating realtime client that invokes functions defined on . /// Include this in a realtime client pipeline to resolve function calls automatically. /// /// /// /// When sessions created by this client receive a in a realtime server message from the inner /// , they respond by invoking the corresponding defined -/// in (or in ), producing a +/// in (or in ), producing a /// that is sent back to the inner session. This loop is repeated until there are no more function calls to make, or until /// another stop condition is met, such as hitting . /// diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs index 14704b48f8a..2f54677b8c4 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs @@ -21,14 +21,14 @@ namespace Microsoft.Extensions.AI; /// -/// A delegating realtime session that invokes functions defined on . +/// A delegating realtime session that invokes functions defined on . /// Include this in a realtime session pipeline to resolve function calls automatically. /// /// /// /// When this session receives a in a realtime server message from its inner /// , it responds by invoking the corresponding defined -/// in (or in ), producing a +/// in (or in ), producing a /// that it sends back to the inner session. This loop is repeated until there are no more function calls to make, or until /// another stop condition is met, such as hitting . /// @@ -40,7 +40,7 @@ namespace Microsoft.Extensions.AI; /// /// /// A instance is thread-safe for concurrent use so long as the -/// instances employed as part of the supplied are also safe. +/// instances employed as part of the supplied are also safe. /// The property can be used to control whether multiple function invocation /// requests as part of the same request are invocable concurrently, but even with that set to /// (the default), multiple concurrent requests to this same instance and using the same tools could result in those @@ -155,7 +155,7 @@ public async IAsyncEnumerable GetStreamingResponseAsync( { // Check if this message contains function calls bool hasFunctionCalls = false; - if (message is RealtimeServerResponseOutputItemMessage responseOutputItemMessage && responseOutputItemMessage.Type == RealtimeServerMessageType.ResponseOutputItemDone) + if (message is ResponseOutputItemRealtimeServerMessage responseOutputItemMessage && responseOutputItemMessage.Type == RealtimeServerMessageType.ResponseOutputItemDone) { // Extract function calls from the message functionCallContents ??= []; @@ -203,7 +203,7 @@ public async IAsyncEnumerable GetStreamingResponseAsync( } /// Extracts function calls from a realtime server message. - private static bool ExtractFunctionCalls(RealtimeServerResponseOutputItemMessage message, List functionCallContents) + private static bool ExtractFunctionCalls(ResponseOutputItemRealtimeServerMessage message, List functionCallContents) { if (message.Item is null) { @@ -349,14 +349,14 @@ private List CreateFunctionResultMessages(List GetStreamingResponseAsync( } // Create activity for ResponseDone message for telemetry - if (message is RealtimeServerResponseCreatedMessage responseDoneMsg && + if (message is ResponseCreatedRealtimeServerMessage responseDoneMsg && responseDoneMsg.Type == RealtimeServerMessageType.ResponseDone) { using Activity? responseActivity = CreateAndConfigureActivity(options); @@ -320,7 +320,7 @@ private static void AddOutputMessagesTag(Activity? activity, ListGets the output modality from a server message, if applicable. private static string? GetOutputModality(RealtimeServerMessage message) { - if (message is RealtimeServerOutputTextAudioMessage textAudio) + if (message is OutputTextAudioRealtimeServerMessage textAudio) { if (textAudio.Type == RealtimeServerMessageType.OutputTextDelta || textAudio.Type == RealtimeServerMessageType.OutputTextDone) { @@ -338,7 +338,7 @@ private static void AddOutputMessagesTag(Activity? activity, List 0 }: + case ResponseCreatedRealtimeServerMessage responseCreatedMsg when responseCreatedMsg.Items is { Count: > 0 }: // Only capture items from ResponseCreated, not ResponseDone (which we use for tracing) if (responseCreatedMsg.Type == RealtimeServerMessageType.ResponseCreated) { @@ -807,7 +807,7 @@ private static string SerializeMessages(IEnumerable message private void TraceStreamingResponse( Activity? activity, string? requestModelId, - RealtimeServerResponseCreatedMessage? response, + ResponseCreatedRealtimeServerMessage? response, Exception? error, Stopwatch? stopwatch) { diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs index f6204bd72a4..261b08af841 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs @@ -40,7 +40,7 @@ public void ConversationItemCreateMessage_Constructor_SetsProperties() var contents = new List { new TextContent("Hello") }; var item = new RealtimeConversationItem(contents, "item_1", ChatRole.User); - var message = new RealtimeClientCreateConversationItemMessage(item, "prev_1"); + var message = new CreateConversationItemRealtimeClientMessage(item, "prev_1"); Assert.Same(item, message.Item); Assert.Equal("prev_1", message.PreviousId); @@ -50,7 +50,7 @@ public void ConversationItemCreateMessage_Constructor_SetsProperties() public void ConversationItemCreateMessage_Constructor_PreviousIdDefaults() { var item = new RealtimeConversationItem([new TextContent("Hello")]); - var message = new RealtimeClientCreateConversationItemMessage(item); + var message = new CreateConversationItemRealtimeClientMessage(item); Assert.Same(item, message.Item); Assert.Null(message.PreviousId); @@ -60,7 +60,7 @@ public void ConversationItemCreateMessage_Constructor_PreviousIdDefaults() public void ConversationItemCreateMessage_Properties_Roundtrip() { var item = new RealtimeConversationItem([new TextContent("Hello")]); - var message = new RealtimeClientCreateConversationItemMessage(item); + var message = new CreateConversationItemRealtimeClientMessage(item); var newItem = new RealtimeConversationItem([new TextContent("World")]); message.Item = newItem; @@ -74,7 +74,7 @@ public void ConversationItemCreateMessage_Properties_Roundtrip() public void ConversationItemCreateMessage_InheritsClientMessage() { var item = new RealtimeConversationItem([new TextContent("Hello")]); - var message = new RealtimeClientCreateConversationItemMessage(item) + var message = new CreateConversationItemRealtimeClientMessage(item) { MessageId = "evt_create_1", }; @@ -87,7 +87,7 @@ public void ConversationItemCreateMessage_InheritsClientMessage() public void InputAudioBufferAppendMessage_Constructor_SetsContent() { var audioContent = new DataContent(new byte[] { 1, 2, 3 }, "audio/pcm"); - var message = new RealtimeClientInputAudioBufferAppendMessage(audioContent); + var message = new InputAudioBufferAppendRealtimeClientMessage(audioContent); Assert.Same(audioContent, message.Content); } @@ -96,7 +96,7 @@ public void InputAudioBufferAppendMessage_Constructor_SetsContent() public void InputAudioBufferAppendMessage_Properties_Roundtrip() { var audioContent = new DataContent(new byte[] { 1, 2, 3 }, "audio/pcm"); - var message = new RealtimeClientInputAudioBufferAppendMessage(audioContent); + var message = new InputAudioBufferAppendRealtimeClientMessage(audioContent); var newContent = new DataContent(new byte[] { 4, 5, 6 }, "audio/wav"); message.Content = newContent; @@ -108,7 +108,7 @@ public void InputAudioBufferAppendMessage_Properties_Roundtrip() public void InputAudioBufferAppendMessage_InheritsClientMessage() { var audioContent = new DataContent(new byte[] { 1, 2, 3 }, "audio/pcm"); - var message = new RealtimeClientInputAudioBufferAppendMessage(audioContent) + var message = new InputAudioBufferAppendRealtimeClientMessage(audioContent) { MessageId = "evt_append_1", }; @@ -120,7 +120,7 @@ public void InputAudioBufferAppendMessage_InheritsClientMessage() [Fact] public void InputAudioBufferCommitMessage_Constructor() { - var message = new RealtimeClientInputAudioBufferCommitMessage(); + var message = new InputAudioBufferCommitRealtimeClientMessage(); Assert.IsAssignableFrom(message); Assert.Null(message.MessageId); @@ -129,7 +129,7 @@ public void InputAudioBufferCommitMessage_Constructor() [Fact] public void ResponseCreateMessage_DefaultProperties() { - var message = new RealtimeClientCreateResponseMessage(); + var message = new CreateResponseRealtimeClientMessage(); Assert.Null(message.Items); Assert.Null(message.OutputAudioOptions); @@ -146,7 +146,7 @@ public void ResponseCreateMessage_DefaultProperties() [Fact] public void ResponseCreateMessage_Properties_Roundtrip() { - var message = new RealtimeClientCreateResponseMessage(); + var message = new CreateResponseRealtimeClientMessage(); var items = new List { @@ -183,7 +183,7 @@ public void ResponseCreateMessage_Properties_Roundtrip() [Fact] public void ResponseCreateMessage_InheritsClientMessage() { - var message = new RealtimeClientCreateResponseMessage + var message = new CreateResponseRealtimeClientMessage { MessageId = "evt_resp_1", RawRepresentation = "raw", diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs index 2f9b2a489f7..caa1c0d80b6 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeServerMessageTests.cs @@ -39,7 +39,7 @@ public void RealtimeServerMessage_Properties_Roundtrip() [Fact] public void ErrorMessage_Constructor_SetsType() { - var message = new RealtimeServerErrorMessage(); + var message = new ErrorRealtimeServerMessage(); Assert.Equal(RealtimeServerMessageType.Error, message.Type); } @@ -47,7 +47,7 @@ public void ErrorMessage_Constructor_SetsType() [Fact] public void ErrorMessage_DefaultProperties() { - var message = new RealtimeServerErrorMessage(); + var message = new ErrorRealtimeServerMessage(); Assert.Null(message.Error); Assert.Null(message.OriginatingMessageId); @@ -57,7 +57,7 @@ public void ErrorMessage_DefaultProperties() public void ErrorMessage_Properties_Roundtrip() { var error = new ErrorContent("Test error") { Details = "temperature" }; - var message = new RealtimeServerErrorMessage + var message = new ErrorRealtimeServerMessage { Error = error, OriginatingMessageId = "evt_bad", @@ -74,7 +74,7 @@ public void ErrorMessage_Properties_Roundtrip() [Fact] public void InputAudioTranscriptionMessage_Constructor_SetsType() { - var message = new RealtimeServerInputAudioTranscriptionMessage( + var message = new InputAudioTranscriptionRealtimeServerMessage( RealtimeServerMessageType.InputAudioTranscriptionCompleted); Assert.Equal(RealtimeServerMessageType.InputAudioTranscriptionCompleted, message.Type); @@ -83,7 +83,7 @@ public void InputAudioTranscriptionMessage_Constructor_SetsType() [Fact] public void InputAudioTranscriptionMessage_DefaultProperties() { - var message = new RealtimeServerInputAudioTranscriptionMessage( + var message = new InputAudioTranscriptionRealtimeServerMessage( RealtimeServerMessageType.InputAudioTranscriptionDelta); Assert.Null(message.ContentIndex); @@ -99,7 +99,7 @@ public void InputAudioTranscriptionMessage_Properties_Roundtrip() var usage = new UsageDetails { InputTokenCount = 10, OutputTokenCount = 20 }; var error = new ErrorContent("transcription error"); - var message = new RealtimeServerInputAudioTranscriptionMessage( + var message = new InputAudioTranscriptionRealtimeServerMessage( RealtimeServerMessageType.InputAudioTranscriptionCompleted) { ContentIndex = 0, @@ -120,7 +120,7 @@ public void InputAudioTranscriptionMessage_Properties_Roundtrip() [Fact] public void OutputTextAudioMessage_Constructor_SetsType() { - var message = new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputTextDelta); + var message = new OutputTextAudioRealtimeServerMessage(RealtimeServerMessageType.OutputTextDelta); Assert.Equal(RealtimeServerMessageType.OutputTextDelta, message.Type); } @@ -128,7 +128,7 @@ public void OutputTextAudioMessage_Constructor_SetsType() [Fact] public void OutputTextAudioMessage_DefaultProperties() { - var message = new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputTextDelta); + var message = new OutputTextAudioRealtimeServerMessage(RealtimeServerMessageType.OutputTextDelta); Assert.Null(message.ContentIndex); Assert.Null(message.Text); @@ -141,7 +141,7 @@ public void OutputTextAudioMessage_DefaultProperties() [Fact] public void OutputTextAudioMessage_Properties_Roundtrip() { - var message = new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputTextDone) + var message = new OutputTextAudioRealtimeServerMessage(RealtimeServerMessageType.OutputTextDone) { ContentIndex = 0, Text = "Hello there!", @@ -162,7 +162,7 @@ public void OutputTextAudioMessage_Properties_Roundtrip() [Fact] public void ResponseCreatedMessage_Constructor_SetsType() { - var message = new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseCreated); + var message = new ResponseCreatedRealtimeServerMessage(RealtimeServerMessageType.ResponseCreated); Assert.Equal(RealtimeServerMessageType.ResponseCreated, message.Type); } @@ -170,7 +170,7 @@ public void ResponseCreatedMessage_Constructor_SetsType() [Fact] public void ResponseCreatedMessage_DefaultProperties() { - var message = new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); + var message = new ResponseCreatedRealtimeServerMessage(RealtimeServerMessageType.ResponseDone); Assert.Null(message.OutputAudioOptions); Assert.Null(message.OutputVoice); @@ -197,7 +197,7 @@ public void ResponseCreatedMessage_Properties_Roundtrip() var error = new ErrorContent("response error"); var usage = new UsageDetails { InputTokenCount = 15, OutputTokenCount = 25, TotalTokenCount = 40 }; - var message = new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone) + var message = new ResponseCreatedRealtimeServerMessage(RealtimeServerMessageType.ResponseDone) { OutputAudioOptions = audioFormat, OutputVoice = "alloy", @@ -227,7 +227,7 @@ public void ResponseCreatedMessage_Properties_Roundtrip() [Fact] public void ResponseOutputItemMessage_Constructor_SetsType() { - var message = new RealtimeServerResponseOutputItemMessage(RealtimeServerMessageType.ResponseOutputItemDone); + var message = new ResponseOutputItemRealtimeServerMessage(RealtimeServerMessageType.ResponseOutputItemDone); Assert.Equal(RealtimeServerMessageType.ResponseOutputItemDone, message.Type); } @@ -235,7 +235,7 @@ public void ResponseOutputItemMessage_Constructor_SetsType() [Fact] public void ResponseOutputItemMessage_DefaultProperties() { - var message = new RealtimeServerResponseOutputItemMessage(RealtimeServerMessageType.ResponseOutputItemAdded); + var message = new ResponseOutputItemRealtimeServerMessage(RealtimeServerMessageType.ResponseOutputItemAdded); Assert.Null(message.ResponseId); Assert.Null(message.OutputIndex); @@ -247,7 +247,7 @@ public void ResponseOutputItemMessage_Properties_Roundtrip() { var item = new RealtimeConversationItem([new TextContent("output")], "item_out_1", ChatRole.Assistant); - var message = new RealtimeServerResponseOutputItemMessage(RealtimeServerMessageType.ResponseOutputItemDone) + var message = new ResponseOutputItemRealtimeServerMessage(RealtimeServerMessageType.ResponseOutputItemDone) { ResponseId = "resp_1", OutputIndex = 0, diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientSessionTests.cs index c2037a7ff21..dbe3c7421f9 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientSessionTests.cs @@ -51,7 +51,7 @@ public async Task Options_InitiallyNull() [Fact] public void SessionUpdateMessage_NullOptions_Throws() { - Assert.Throws("options", () => new RealtimeClientSessionUpdateMessage(null!)); + Assert.Throws("options", () => new SessionUpdateRealtimeClientMessage(null!)); } [Fact] diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeClientTests.cs index a3f661ad192..0ea2d47bf8e 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/FunctionInvokingRealtimeClientTests.cs @@ -124,14 +124,14 @@ public async Task GetStreamingResponseAsync_FunctionCall_InvokesAndInjectsResult Assert.Equal(2, injectedMessages.Count); // First injected: conversation.item.create with function result - var resultMsg = Assert.IsType(injectedMessages[0]); + var resultMsg = Assert.IsType(injectedMessages[0]); Assert.NotNull(resultMsg.Item); var functionResult = Assert.IsType(resultMsg.Item.Contents[0]); Assert.Equal("call_001", functionResult.CallId); Assert.Contains("Sunny in Seattle", functionResult.Result?.ToString()); // Second injected: response.create (no hardcoded modalities) - var responseCreate = Assert.IsType(injectedMessages[1]); + var responseCreate = Assert.IsType(injectedMessages[1]); Assert.Null(responseCreate.OutputModalities); } @@ -167,7 +167,7 @@ public async Task GetStreamingResponseAsync_FunctionCall_FromAdditionalTools() } Assert.Equal(2, injectedMessages.Count); - var resultMsg = Assert.IsType(injectedMessages[0]); + var resultMsg = Assert.IsType(injectedMessages[0]); var functionResult = Assert.IsType(resultMsg.Item.Contents[0]); Assert.Contains("Rainy in London", functionResult.Result?.ToString()); } @@ -274,7 +274,7 @@ public async Task GetStreamingResponseAsync_UnknownFunction_SendsErrorByDefault( // Should inject error result + response.create Assert.Equal(2, injectedMessages.Count); - var resultMsg = Assert.IsType(injectedMessages[0]); + var resultMsg = Assert.IsType(injectedMessages[0]); var functionResult = Assert.IsType(resultMsg.Item.Contents[0]); Assert.Contains("not found", functionResult.Result?.ToString(), StringComparison.OrdinalIgnoreCase); } @@ -312,7 +312,7 @@ public async Task GetStreamingResponseAsync_FunctionError_IncludesDetailedErrors } Assert.Equal(2, injectedMessages.Count); - var resultMsg = Assert.IsType(injectedMessages[0]); + var resultMsg = Assert.IsType(injectedMessages[0]); var functionResult = Assert.IsType(resultMsg.Item.Contents[0]); Assert.Contains("Something broke", functionResult.Result?.ToString()); } @@ -349,7 +349,7 @@ public async Task GetStreamingResponseAsync_FunctionError_HidesDetailsWhenNotEna // consume } - var resultMsg = Assert.IsType(injectedMessages[0]); + var resultMsg = Assert.IsType(injectedMessages[0]); var functionResult = Assert.IsType(resultMsg.Item.Contents[0]); Assert.DoesNotContain("Secret error info", functionResult.Result?.ToString()); Assert.Contains("failed", functionResult.Result?.ToString(), StringComparison.OrdinalIgnoreCase); @@ -433,7 +433,7 @@ public async Task GetStreamingResponseAsync_TerminateOnUnknownCalls_False_SendsE // Error result + response.create should be injected (default behavior) Assert.Equal(2, injectedMessages.Count); - var resultMsg = Assert.IsType(injectedMessages[0]); + var resultMsg = Assert.IsType(injectedMessages[0]); var functionResult = Assert.IsType(resultMsg.Item.Contents[0]); Assert.Contains("not found", functionResult.Result?.ToString(), StringComparison.OrdinalIgnoreCase); } @@ -482,7 +482,7 @@ public async Task GetStreamingResponseAsync_ConcurrentInvocation_InvokesInParall new FunctionCallContent("call_b", "slow_func"), ], "item_combined"); - var combinedMessage = new RealtimeServerResponseOutputItemMessage(RealtimeServerMessageType.ResponseOutputItemDone) + var combinedMessage = new ResponseOutputItemRealtimeServerMessage(RealtimeServerMessageType.ResponseOutputItemDone) { ResponseId = "resp_combined", OutputIndex = 0, @@ -635,13 +635,13 @@ private static FunctionInvokingRealtimeClient CreateClient(IRealtimeClientSessio } #pragma warning restore CA2000 - private static RealtimeServerResponseOutputItemMessage CreateFunctionCallOutputItemMessage( + private static ResponseOutputItemRealtimeServerMessage CreateFunctionCallOutputItemMessage( string callId, string functionName, IDictionary? arguments) { var functionCallContent = new FunctionCallContent(callId, functionName, arguments); var item = new RealtimeConversationItem([functionCallContent], $"item_{callId}"); - return new RealtimeServerResponseOutputItemMessage(RealtimeServerMessageType.ResponseOutputItemDone) + return new ResponseOutputItemRealtimeServerMessage(RealtimeServerMessageType.ResponseOutputItemDone) { ResponseId = $"resp_{callId}", OutputIndex = 0, diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeClientTests.cs index 86f5e76956e..0eac43d40bc 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/LoggingRealtimeClientTests.cs @@ -80,7 +80,7 @@ public async Task SendAsync_SessionUpdateMessage_LogsInvocationAndCompletion(Log .Build(services); await using var session = await client.CreateSessionAsync(); - await session.SendAsync(new RealtimeClientSessionUpdateMessage(new RealtimeSessionOptions { Model = "test-model", Instructions = "Be helpful" })); + await session.SendAsync(new SessionUpdateRealtimeClientMessage(new RealtimeSessionOptions { Model = "test-model", Instructions = "Be helpful" })); var logs = collector.GetSnapshot(); if (level is LogLevel.Trace) @@ -226,7 +226,7 @@ public async Task SendAsync_SessionUpdateMessage_LogsCancellation() await using var session = await client.CreateSessionAsync(); cts.Cancel(); - await Assert.ThrowsAsync(() => session.SendAsync(new RealtimeClientSessionUpdateMessage(new RealtimeSessionOptions()), cts.Token)); + await Assert.ThrowsAsync(() => session.SendAsync(new SessionUpdateRealtimeClientMessage(new RealtimeSessionOptions()), cts.Token)); var logs = collector.GetSnapshot(); Assert.Collection(logs, @@ -255,7 +255,7 @@ public async Task SendAsync_SessionUpdateMessage_LogsErrors() .Build(); await using var session = await client.CreateSessionAsync(); - await Assert.ThrowsAsync(() => session.SendAsync(new RealtimeClientSessionUpdateMessage(new RealtimeSessionOptions()))); + await Assert.ThrowsAsync(() => session.SendAsync(new SessionUpdateRealtimeClientMessage(new RealtimeSessionOptions()))); var logs = collector.GetSnapshot(); Assert.Collection(logs, diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientTests.cs index 69865f5dec6..13397e99eed 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientTests.cs @@ -52,11 +52,11 @@ static async IAsyncEnumerable CallbackAsync([EnumeratorCa _ = cancellationToken; yield return new RealtimeServerMessage { Type = RealtimeServerMessageType.ResponseCreated, MessageId = "evt_001" }; - yield return new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputTextDelta) { OutputIndex = 0, Text = "Hello" }; - yield return new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputTextDelta) { OutputIndex = 0, Text = " there!" }; - yield return new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputTextDone) { OutputIndex = 0, Text = "Hello there!" }; + yield return new OutputTextAudioRealtimeServerMessage(RealtimeServerMessageType.OutputTextDelta) { OutputIndex = 0, Text = "Hello" }; + yield return new OutputTextAudioRealtimeServerMessage(RealtimeServerMessageType.OutputTextDelta) { OutputIndex = 0, Text = " there!" }; + yield return new OutputTextAudioRealtimeServerMessage(RealtimeServerMessageType.OutputTextDone) { OutputIndex = 0, Text = "Hello there!" }; - yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone) + yield return new ResponseCreatedRealtimeServerMessage(RealtimeServerMessageType.ResponseDone) { ResponseId = "resp_12345", Status = "completed", @@ -243,7 +243,7 @@ static async IAsyncEnumerable ErrorResponseCallbackAsync( await Task.Yield(); _ = cancellationToken; - yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone) + yield return new ResponseCreatedRealtimeServerMessage(RealtimeServerMessageType.ResponseDone) { ResponseId = "resp_error", Status = "failed", @@ -331,7 +331,7 @@ public async Task InvalidArgs_Throws() [Fact] public void SessionUpdateMessage_NullOptions_Throws() { - Assert.Throws("options", () => new RealtimeClientSessionUpdateMessage(null!)); + Assert.Throws("options", () => new SessionUpdateRealtimeClientMessage(null!)); } [Fact] @@ -385,11 +385,11 @@ static async IAsyncEnumerable TranscriptionCallbackAsync( await Task.Yield(); _ = cancellationToken; - yield return new RealtimeServerInputAudioTranscriptionMessage(RealtimeServerMessageType.InputAudioTranscriptionCompleted) + yield return new InputAudioTranscriptionRealtimeServerMessage(RealtimeServerMessageType.InputAudioTranscriptionCompleted) { Transcription = "Hello world", }; - yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); + yield return new ResponseCreatedRealtimeServerMessage(RealtimeServerMessageType.ResponseDone); } using var innerClient = new TestRealtimeClient(innerSession); @@ -721,7 +721,7 @@ private static async IAsyncEnumerable SimpleCallbackAsync await Task.Yield(); _ = cancellationToken; - yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); + yield return new ResponseCreatedRealtimeServerMessage(RealtimeServerMessageType.ResponseDone); } #pragma warning disable IDE0060 // Remove unused parameter @@ -729,9 +729,9 @@ private static async IAsyncEnumerable GetClientMessagesAs #pragma warning restore IDE0060 { await Task.Yield(); - yield return new RealtimeClientInputAudioBufferAppendMessage(new DataContent(new byte[] { 1, 2, 3 }, "audio/pcm")); - yield return new RealtimeClientInputAudioBufferCommitMessage(); - yield return new RealtimeClientCreateResponseMessage(); + yield return new InputAudioBufferAppendRealtimeClientMessage(new DataContent(new byte[] { 1, 2, 3 }, "audio/pcm")); + yield return new InputAudioBufferCommitRealtimeClientMessage(); + yield return new CreateResponseRealtimeClientMessage(); } #pragma warning disable IDE0060 // Remove unused parameter @@ -740,8 +740,8 @@ private static async IAsyncEnumerable GetClientMessagesWi { await Task.Yield(); var contentItem = new RealtimeConversationItem([new FunctionResultContent("call_1", "result_value")], role: ChatRole.Tool); - yield return new RealtimeClientCreateConversationItemMessage(contentItem); - yield return new RealtimeClientCreateResponseMessage(); + yield return new CreateConversationItemRealtimeClientMessage(contentItem); + yield return new CreateResponseRealtimeClientMessage(); } private static async IAsyncEnumerable CallbackWithToolCallAsync([EnumeratorCancellation] CancellationToken cancellationToken) @@ -749,16 +749,16 @@ private static async IAsyncEnumerable CallbackWithToolCal await Task.Yield(); _ = cancellationToken; - // Yield a function call item from the server using RealtimeServerResponseOutputItemMessage + // Yield a function call item from the server using ResponseOutputItemRealtimeServerMessage var contentItem = new RealtimeConversationItem( [new FunctionCallContent("call_123", "search", new Dictionary { ["query"] = "test" })], role: ChatRole.Assistant); - yield return new RealtimeServerResponseOutputItemMessage(RealtimeServerMessageType.ResponseOutputItemDone) + yield return new ResponseOutputItemRealtimeServerMessage(RealtimeServerMessageType.ResponseOutputItemDone) { Item = contentItem, }; - yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); + yield return new ResponseCreatedRealtimeServerMessage(RealtimeServerMessageType.ResponseDone); } [Fact] @@ -1142,7 +1142,7 @@ private static async IAsyncEnumerable GetClientMessagesWi #pragma warning restore IDE0060 { await Task.Yield(); - yield return new RealtimeClientCreateResponseMessage { Instructions = "Be very helpful" }; + yield return new CreateResponseRealtimeClientMessage { Instructions = "Be very helpful" }; } #pragma warning disable IDE0060 // Remove unused parameter @@ -1151,7 +1151,7 @@ private static async IAsyncEnumerable GetClientMessagesWi { await Task.Yield(); var item = new RealtimeConversationItem([new TextContent("Hello from client")], role: ChatRole.User); - yield return new RealtimeClientCreateResponseMessage { Items = [item] }; + yield return new CreateResponseRealtimeClientMessage { Items = [item] }; } #pragma warning disable IDE0060 // Remove unused parameter @@ -1160,8 +1160,8 @@ private static async IAsyncEnumerable GetClientMessagesWi { await Task.Yield(); var item = new RealtimeConversationItem([new TextContent("User text message")], role: ChatRole.User); - yield return new RealtimeClientCreateConversationItemMessage(item); - yield return new RealtimeClientCreateResponseMessage(); + yield return new CreateConversationItemRealtimeClientMessage(item); + yield return new CreateResponseRealtimeClientMessage(); } #pragma warning disable IDE0060 // Remove unused parameter @@ -1171,8 +1171,8 @@ private static async IAsyncEnumerable GetClientMessagesWi await Task.Yield(); var imageData = new DataContent(new byte[] { 0x89, 0x50, 0x4E, 0x47 }, "image/png"); var item = new RealtimeConversationItem([imageData], role: ChatRole.User); - yield return new RealtimeClientCreateConversationItemMessage(item); - yield return new RealtimeClientCreateResponseMessage(); + yield return new CreateConversationItemRealtimeClientMessage(item); + yield return new CreateResponseRealtimeClientMessage(); } private static async IAsyncEnumerable CallbackWithTextOutputAsync([EnumeratorCancellation] CancellationToken cancellationToken) @@ -1180,11 +1180,11 @@ private static async IAsyncEnumerable CallbackWithTextOut await Task.Yield(); _ = cancellationToken; - yield return new RealtimeServerOutputTextAudioMessage(RealtimeServerMessageType.OutputTextDone) + yield return new OutputTextAudioRealtimeServerMessage(RealtimeServerMessageType.OutputTextDone) { Text = "Hello from server", }; - yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); + yield return new ResponseCreatedRealtimeServerMessage(RealtimeServerMessageType.ResponseDone); } private static async IAsyncEnumerable CallbackWithTranscriptionAsync([EnumeratorCancellation] CancellationToken cancellationToken) @@ -1192,11 +1192,11 @@ private static async IAsyncEnumerable CallbackWithTranscr await Task.Yield(); _ = cancellationToken; - yield return new RealtimeServerInputAudioTranscriptionMessage(RealtimeServerMessageType.InputAudioTranscriptionCompleted) + yield return new InputAudioTranscriptionRealtimeServerMessage(RealtimeServerMessageType.InputAudioTranscriptionCompleted) { Transcription = "Transcribed audio content", }; - yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); + yield return new ResponseCreatedRealtimeServerMessage(RealtimeServerMessageType.ResponseDone); } private static async IAsyncEnumerable CallbackWithServerErrorAsync([EnumeratorCancellation] CancellationToken cancellationToken) @@ -1204,11 +1204,11 @@ private static async IAsyncEnumerable CallbackWithServerE await Task.Yield(); _ = cancellationToken; - yield return new RealtimeServerErrorMessage + yield return new ErrorRealtimeServerMessage { Error = new ErrorContent("Something went wrong on server"), }; - yield return new RealtimeServerResponseCreatedMessage(RealtimeServerMessageType.ResponseDone); + yield return new ResponseCreatedRealtimeServerMessage(RealtimeServerMessageType.ResponseDone); } private static string ReplaceWhitespace(string? input) => Regex.Replace(input ?? "", @"\s+", " ").Trim(); From b8d57419c82d670c3412d722b9df95c30569e4d3 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Fri, 6 Mar 2026 14:54:56 -0800 Subject: [PATCH 70/92] Fix cancellation tests to also accept WebSocketException on net462 --- .../OpenAIRealtimeClientSessionTests.cs | 5 ++++- .../OpenAIRealtimeClientTests.cs | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientSessionTests.cs index dbe3c7421f9..c98a99e8d16 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientSessionTests.cs @@ -80,6 +80,9 @@ public async Task ConnectAsync_CancelledToken_Throws() using var cts = new CancellationTokenSource(); cts.Cancel(); - await Assert.ThrowsAnyAsync(() => session.ConnectAsync(cts.Token)); + var ex = await Assert.ThrowsAnyAsync(() => session.ConnectAsync(cts.Token)); + Assert.True( + ex is OperationCanceledException || ex is System.Net.WebSockets.WebSocketException, + $"Expected OperationCanceledException or WebSocketException but got {ex.GetType().FullName}"); } } diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientTests.cs index 0f39c004f31..3167f7da7e0 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientTests.cs @@ -55,6 +55,9 @@ public async Task CreateSessionAsync_Cancelled_Throws() using var cts = new CancellationTokenSource(); cts.Cancel(); - await Assert.ThrowsAnyAsync(() => client.CreateSessionAsync(cancellationToken: cts.Token)); + var ex = await Assert.ThrowsAnyAsync(() => client.CreateSessionAsync(cancellationToken: cts.Token)); + Assert.True( + ex is OperationCanceledException || ex is System.Net.WebSockets.WebSocketException, + $"Expected OperationCanceledException or WebSocketException but got {ex.GetType().FullName}"); } } From 51213de5f03470478a69d9613c83c7f556978e53 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Fri, 6 Mar 2026 16:41:11 -0800 Subject: [PATCH 71/92] Remove PreviousId from CreateConversationItemRealtimeClientMessage as OpenAI-specific --- .../CreateConversationItemRealtimeClientMessage.cs | 10 +--------- .../OpenAIRealtimeClientSession.cs | 10 ++++++++-- .../Realtime/RealtimeClientMessageTests.cs | 13 ------------- 3 files changed, 9 insertions(+), 24 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateConversationItemRealtimeClientMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateConversationItemRealtimeClientMessage.cs index 20c6288ac0a..0f1f245e00c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateConversationItemRealtimeClientMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateConversationItemRealtimeClientMessage.cs @@ -17,19 +17,11 @@ public class CreateConversationItemRealtimeClientMessage : RealtimeClientMessage /// Initializes a new instance of the class. /// /// The conversation item to create. - /// The optional ID of the previous conversation item to insert the new one after. - public CreateConversationItemRealtimeClientMessage(RealtimeConversationItem item, string? previousId = null) + public CreateConversationItemRealtimeClientMessage(RealtimeConversationItem item) { - PreviousId = previousId; Item = Throw.IfNull(item); } - /// - /// Gets or sets the optional previous conversation item ID. - /// If not set, the new item will be appended to the end of the conversation. - /// - public string? PreviousId { get; set; } - /// /// Gets or sets the conversation item to create. /// diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs index 1c24f084b2f..9265f4dc0e4 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs @@ -318,12 +318,18 @@ private async Task SendConversationItemCreateAsync(CreateConversationItemRealtim return; } - if (itemCreate.MessageId is not null || itemCreate.PreviousId is not null) + string? previousId = null; + if (itemCreate.RawRepresentation is Sdk.RealtimeClientCommandConversationItemCreate rawCmd) + { + previousId = rawCmd.PreviousItemId; + } + + if (itemCreate.MessageId is not null || previousId is not null) { var cmd = new Sdk.RealtimeClientCommandConversationItemCreate(sdkItem) { EventId = itemCreate.MessageId, - PreviousItemId = itemCreate.PreviousId, + PreviousItemId = previousId, }; await _sessionClient!.SendCommandAsync(cmd, cancellationToken).ConfigureAwait(false); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs index 261b08af841..0ca99aa3495 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeClientMessageTests.cs @@ -40,20 +40,9 @@ public void ConversationItemCreateMessage_Constructor_SetsProperties() var contents = new List { new TextContent("Hello") }; var item = new RealtimeConversationItem(contents, "item_1", ChatRole.User); - var message = new CreateConversationItemRealtimeClientMessage(item, "prev_1"); - - Assert.Same(item, message.Item); - Assert.Equal("prev_1", message.PreviousId); - } - - [Fact] - public void ConversationItemCreateMessage_Constructor_PreviousIdDefaults() - { - var item = new RealtimeConversationItem([new TextContent("Hello")]); var message = new CreateConversationItemRealtimeClientMessage(item); Assert.Same(item, message.Item); - Assert.Null(message.PreviousId); } [Fact] @@ -64,10 +53,8 @@ public void ConversationItemCreateMessage_Properties_Roundtrip() var newItem = new RealtimeConversationItem([new TextContent("World")]); message.Item = newItem; - message.PreviousId = "prev_2"; Assert.Same(newItem, message.Item); - Assert.Equal("prev_2", message.PreviousId); } [Fact] From 056f19b09f22533483935fae9d2e06449f7d6e4c Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Fri, 6 Mar 2026 16:58:28 -0800 Subject: [PATCH 72/92] Document that ResponseId may be null and response lifecycle events may be synthesized --- .../Realtime/OutputTextAudioRealtimeServerMessage.cs | 3 +++ .../Realtime/ResponseCreatedRealtimeServerMessage.cs | 10 ++++++++++ .../ResponseOutputItemRealtimeServerMessage.cs | 3 +++ 3 files changed, 16 insertions(+) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/OutputTextAudioRealtimeServerMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/OutputTextAudioRealtimeServerMessage.cs index 60eab81aedf..37861c4f76e 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/OutputTextAudioRealtimeServerMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/OutputTextAudioRealtimeServerMessage.cs @@ -66,5 +66,8 @@ public OutputTextAudioRealtimeServerMessage(RealtimeServerMessageType type) /// /// Gets or sets the ID of the response. /// + /// + /// May be for providers that do not natively track response lifecycle. + /// public string? ResponseId { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ResponseCreatedRealtimeServerMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ResponseCreatedRealtimeServerMessage.cs index d40f0ee0433..eff291abd8f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ResponseCreatedRealtimeServerMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ResponseCreatedRealtimeServerMessage.cs @@ -20,6 +20,11 @@ namespace Microsoft.Extensions.AI; /// when the response is complete. The built-in middleware depends /// on these messages for tracing response lifecycle. /// +/// +/// Providers that do not natively support response lifecycle events (e.g., those that only stream content parts +/// and signal turn completion) should synthesize these messages to ensure correct middleware behavior. +/// In such cases, may be set to a synthetic value or left . +/// /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] public class ResponseCreatedRealtimeServerMessage : RealtimeServerMessage @@ -48,6 +53,11 @@ public ResponseCreatedRealtimeServerMessage(RealtimeServerMessageType type) /// /// Gets or sets the unique response ID. /// + /// + /// Some providers (e.g., OpenAI) assign a unique ID to each response. Providers that do not + /// natively track response lifecycles may set this to or generate a synthetic ID. + /// Consumers should not assume this value correlates to a provider-specific concept. + /// public string? ResponseId { get; set; } /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ResponseOutputItemRealtimeServerMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ResponseOutputItemRealtimeServerMessage.cs index 24ee1dfc2af..bd2d5ecbafb 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ResponseOutputItemRealtimeServerMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ResponseOutputItemRealtimeServerMessage.cs @@ -37,6 +37,9 @@ public ResponseOutputItemRealtimeServerMessage(RealtimeServerMessageType type) /// /// Gets or sets the unique response ID. /// + /// + /// May be for providers that do not natively track response lifecycle. + /// public string? ResponseId { get; set; } /// From 2075355f83387bd292383cb352f71e20ab3f306a Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Fri, 6 Mar 2026 17:01:53 -0800 Subject: [PATCH 73/92] Document that CreateResponseRealtimeClientMessage may be a no-op for VAD-driven providers --- .../Realtime/CreateResponseRealtimeClientMessage.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateResponseRealtimeClientMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateResponseRealtimeClientMessage.cs index 0f4498a8f83..4ee2bc36cb1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateResponseRealtimeClientMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateResponseRealtimeClientMessage.cs @@ -11,10 +11,18 @@ namespace Microsoft.Extensions.AI; /// Represents a client message that triggers model inference to generate a response. /// /// +/// /// Sending this message instructs the provider to generate a new response from the model. /// The response may include one or more output items (text, audio, or tool calls). /// Properties on this message optionally override the session-level configuration /// for this response only. +/// +/// +/// Not all providers support explicit response triggering. Voice-activity-detection (VAD) driven +/// providers may respond automatically when speech is detected or input is committed, in which case +/// this message may be treated as a no-op. Per-response overrides (instructions, tools, voice, etc.) +/// are advisory and may be silently ignored by providers that do not support them. +/// /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] public class CreateResponseRealtimeClientMessage : RealtimeClientMessage From 712eb7eac2f4acd1dc49cfb55b09cb2025a3e650 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Fri, 6 Mar 2026 17:25:58 -0800 Subject: [PATCH 74/92] Add RealtimeResponseStatus constants and document interruption/barge-in via Status property --- .../Realtime/RealtimeResponseStatus.cs | 42 +++++++++++++++++++ .../ResponseCreatedRealtimeServerMessage.cs | 7 ++++ 2 files changed, 49 insertions(+) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeResponseStatus.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeResponseStatus.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeResponseStatus.cs new file mode 100644 index 00000000000..133bcabec79 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeResponseStatus.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// +/// Defines well-known status values for real-time response lifecycle messages. +/// +/// +/// These constants represent the standard status values that may appear on +/// when the response completes +/// (i.e., on ). +/// Providers may use additional status values beyond those defined here. +/// +[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] +public static class RealtimeResponseStatus +{ + /// + /// The response completed successfully. + /// + public const string Completed = "completed"; + + /// + /// The response was cancelled, typically due to an interruption such as user barge-in + /// (the user started speaking while the model was generating output). + /// + public const string Cancelled = "cancelled"; + + /// + /// The response ended before completing, for example because the output reached + /// the maximum token limit. + /// + public const string Incomplete = "incomplete"; + + /// + /// The response failed due to an error. + /// + public const string Failed = "failed"; +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ResponseCreatedRealtimeServerMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ResponseCreatedRealtimeServerMessage.cs index eff291abd8f..517b9f8dc04 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ResponseCreatedRealtimeServerMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/ResponseCreatedRealtimeServerMessage.cs @@ -93,6 +93,13 @@ public ResponseCreatedRealtimeServerMessage(RealtimeServerMessageType type) /// /// Gets or sets the status of the response. /// + /// + /// Typically set on messages to indicate + /// how the response ended. See for well-known values + /// such as , + /// (e.g., due to user barge-in), , + /// and . + /// public string? Status { get; set; } /// From 046a71bbb1cbc48ea1ad97a13c9057fa5889c3a1 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Fri, 6 Mar 2026 17:41:19 -0800 Subject: [PATCH 75/92] Remove session parameter from RealtimeSessionOptions.RawRepresentationFactory --- .../Realtime/RealtimeSessionOptions.cs | 7 ++++++- .../OpenAIRealtimeClientSession.cs | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs index 0866324e664..e4aa12e9a1c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs @@ -88,7 +88,12 @@ public class RealtimeSessionOptions /// a new instance on each call. /// This is typically used to set an implementation-specific setting that isn't otherwise exposed from the strongly typed /// properties on . + /// + /// Unlike similar factories on other options types, this callback does not receive the session instance + /// as a parameter because some providers need to evaluate it before the session is created + /// (e.g., to produce connection configuration). + /// /// [JsonIgnore] - public Func? RawRepresentationFactory { get; init; } + public Func? RawRepresentationFactory { get; init; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs index 9265f4dc0e4..5a78e99e6e7 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs @@ -85,7 +85,7 @@ private async Task UpdateSessionAsync(RealtimeSessionOptions options, Cancellati if (_sessionClient is not null) { // Allow callers to provide a pre-configured SDK-specific options instance. - object? rawOptions = options.RawRepresentationFactory?.Invoke(this); + object? rawOptions = options.RawRepresentationFactory?.Invoke(); if (rawOptions is Sdk.RealtimeTranscriptionSessionOptions rawTransOptions) { From 6d4eddeb1d356247112909133b45dd7c6f727485 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Mon, 9 Mar 2026 09:42:40 -0700 Subject: [PATCH 76/92] Fix AOT compatibility errors in OpenAIRealtimeClientSession by using source-generated JSON serialization --- .../OpenAIJsonContext.cs | 1 + .../OpenAIRealtimeClientSession.cs | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIJsonContext.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIJsonContext.cs index 9a040864613..fcdf957762b 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIJsonContext.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIJsonContext.cs @@ -17,6 +17,7 @@ namespace Microsoft.Extensions.AI; WriteIndented = true)] [JsonSerializable(typeof(OpenAIClientExtensions.ToolJson))] [JsonSerializable(typeof(IDictionary))] +[JsonSerializable(typeof(string))] [JsonSerializable(typeof(string[]))] [JsonSerializable(typeof(IEnumerable))] [JsonSerializable(typeof(JsonElement))] diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs index 5a78e99e6e7..dbccaf06bb1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs @@ -379,7 +379,7 @@ private async Task SendRawCommandAsync(RealtimeClientMessage message, Cancellati // Inject event_id if the message has one but the raw JSON does not. if (message.MessageId is not null && !jsonString.Contains("\"event_id\"", StringComparison.Ordinal)) { - jsonString = jsonString.Insert(1, $"\"event_id\":{JsonSerializer.Serialize(message.MessageId)},"); + jsonString = jsonString.Insert(1, $"\"event_id\":{JsonSerializer.Serialize(message.MessageId, OpenAIJsonContext.Default.String)},"); } await _sessionClient!.SendCommandAsync(BinaryData.FromString(jsonString), null).ConfigureAwait(false); @@ -601,15 +601,18 @@ private static Sdk.RealtimeMcpToolCallApprovalPolicy ToSdkCustomApprovalPolicy(H if (firstContent is FunctionResultContent functionResult) { + string resultJson = functionResult.Result is not null + ? JsonSerializer.Serialize(functionResult.Result, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(object))) + : string.Empty; return Sdk.RealtimeItem.CreateFunctionCallOutputItem( functionResult.CallId ?? string.Empty, - functionResult.Result is not null ? JsonSerializer.Serialize(functionResult.Result) : string.Empty); + resultJson); } if (firstContent is FunctionCallContent functionCall) { var arguments = functionCall.Arguments is not null - ? BinaryData.FromString(JsonSerializer.Serialize(functionCall.Arguments)) + ? BinaryData.FromString(JsonSerializer.Serialize(functionCall.Arguments, OpenAIJsonContext.Default.IDictionaryStringObject)) : BinaryData.FromString("{}"); return Sdk.RealtimeItem.CreateFunctionCallItem( functionCall.CallId ?? string.Empty, @@ -1074,7 +1077,7 @@ [new FunctionResultContent(funcOutputItem.CallId ?? string.Empty, funcOutputItem private static RealtimeConversationItem MapFunctionCallItem(Sdk.RealtimeFunctionCallItem funcCallItem) { var arguments = funcCallItem.FunctionArguments is not null && !funcCallItem.FunctionArguments.IsEmpty - ? JsonSerializer.Deserialize>(funcCallItem.FunctionArguments) + ? JsonSerializer.Deserialize(funcCallItem.FunctionArguments, OpenAIJsonContext.Default.IDictionaryStringObject) : null; return new RealtimeConversationItem( [new FunctionCallContent(funcCallItem.CallId ?? string.Empty, funcCallItem.FunctionName, arguments)], @@ -1140,7 +1143,7 @@ private static RealtimeConversationItem MapMcpToolCallItem(Sdk.RealtimeMcpToolCa string argsJson = mcpItem.ToolArguments.ToString(); if (!string.IsNullOrEmpty(argsJson)) { - arguments = JsonSerializer.Deserialize>(argsJson); + arguments = JsonSerializer.Deserialize(argsJson, OpenAIJsonContext.Default.IDictionaryStringObject); } } @@ -1179,7 +1182,7 @@ private static RealtimeConversationItem MapMcpApprovalRequestItem(Sdk.RealtimeMc string argsJson = approvalItem.ToolArguments.ToString(); if (!string.IsNullOrEmpty(argsJson)) { - arguments = JsonSerializer.Deserialize>(argsJson); + arguments = JsonSerializer.Deserialize(argsJson, OpenAIJsonContext.Default.IDictionaryStringObject); } } From b8a7c4a42ab8405fafe3471d07997a8bf2815f45 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Mon, 9 Mar 2026 11:31:55 -0700 Subject: [PATCH 77/92] Fix SpeechToTextOptionsTests for unified TranscriptionOptions after upstream merge --- .../SpeechToText/SpeechToTextOptionsTests.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/SpeechToText/SpeechToTextOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/SpeechToText/SpeechToTextOptionsTests.cs index c115464aac7..ec04bbbfffa 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/SpeechToText/SpeechToTextOptionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/SpeechToText/SpeechToTextOptionsTests.cs @@ -157,8 +157,10 @@ public void JsonDeserialization_KnownPayload() { const string Json = """ { - "modelId": "whisper-1", - "speechLanguage": "en-US", + "transcription": { + "modelId": "whisper-1", + "speechLanguage": "en-US" + }, "speechSampleRate": 16000, "textLanguage": "en", "additionalProperties": { @@ -170,8 +172,9 @@ public void JsonDeserialization_KnownPayload() SpeechToTextOptions? result = JsonSerializer.Deserialize(Json, AIJsonUtilities.DefaultOptions); Assert.NotNull(result); - Assert.Equal("whisper-1", result.ModelId); - Assert.Equal("en-US", result.SpeechLanguage); + Assert.NotNull(result.Transcription); + Assert.Equal("whisper-1", result.Transcription.ModelId); + Assert.Equal("en-US", result.Transcription.SpeechLanguage); Assert.Equal(16000, result.SpeechSampleRate); Assert.Equal("en", result.TextLanguage); Assert.NotNull(result.AdditionalProperties); From 6ff1f88e84b9d5cf2b6c48e56d65f2a8ea92cf18 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 10 Mar 2026 12:54:26 -0700 Subject: [PATCH 78/92] Revert SpeechToTextOptions changes per review feedback Restore ModelId and SpeechLanguage as direct properties on SpeechToTextOptions instead of nesting them under a Transcription property. TranscriptionOptions class remains for RealtimeSessionOptions. - Restore SpeechToTextOptions.ModelId and .SpeechLanguage properties - Remove SpeechToTextOptions.Transcription property - Update all consumers to use direct property access - Fix SpeechToTextClientMetadata doc reference - Regenerate CompatibilitySuppressions.xml - Update all related tests --- .../CompatibilitySuppressions.xml | 215 ++++-------------- .../SpeechToTextClientMetadata.cs | 2 +- .../SpeechToText/SpeechToTextOptions.cs | 10 +- .../OpenAISpeechToTextClient.cs | 6 +- .../OpenTelemetrySpeechToTextClient.cs | 6 +- .../SpeechToText/SpeechToTextOptionsTests.cs | 41 ++-- .../OpenAISpeechToTextClientTests.cs | 8 +- ...ConfigureOptionsSpeechToTextClientTests.cs | 6 +- .../LoggingSpeechToTextClientTests.cs | 4 +- .../OpenTelemetrySpeechToTextClientTests.cs | 2 +- 10 files changed, 82 insertions(+), 218 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/CompatibilitySuppressions.xml b/src/Libraries/Microsoft.Extensions.AI.Abstractions/CompatibilitySuppressions.xml index 98d238db9ad..8c91fd44812 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/CompatibilitySuppressions.xml +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/CompatibilitySuppressions.xml @@ -1,5 +1,6 @@ - - + + + CP0001 T:Microsoft.Extensions.AI.FunctionApprovalRequestContent @@ -14,6 +15,13 @@ lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll true + + CP0001 + T:Microsoft.Extensions.AI.IToolReductionStrategy + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll + true + CP0001 T:Microsoft.Extensions.AI.McpServerToolApprovalRequestContent @@ -56,6 +64,13 @@ lib/net462/Microsoft.Extensions.AI.Abstractions.dll true + + CP0001 + T:Microsoft.Extensions.AI.IToolReductionStrategy + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + lib/net462/Microsoft.Extensions.AI.Abstractions.dll + true + CP0001 T:Microsoft.Extensions.AI.McpServerToolApprovalRequestContent @@ -98,6 +113,13 @@ lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll true + + CP0001 + T:Microsoft.Extensions.AI.IToolReductionStrategy + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll + true + CP0001 T:Microsoft.Extensions.AI.McpServerToolApprovalRequestContent @@ -140,6 +162,13 @@ lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll true + + CP0001 + T:Microsoft.Extensions.AI.IToolReductionStrategy + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll + true + CP0001 T:Microsoft.Extensions.AI.McpServerToolApprovalRequestContent @@ -182,6 +211,13 @@ lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll true + + CP0001 + T:Microsoft.Extensions.AI.IToolReductionStrategy + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll + true + CP0001 T:Microsoft.Extensions.AI.McpServerToolApprovalRequestContent @@ -770,179 +806,4 @@ lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll true - - CP0001 - T:Microsoft.Extensions.AI.IToolReductionStrategy - lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll - lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.AI.IToolReductionStrategy - lib/net462/Microsoft.Extensions.AI.Abstractions.dll - lib/net462/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.AI.IToolReductionStrategy - lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll - lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.AI.IToolReductionStrategy - lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll - lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.AI.IToolReductionStrategy - lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId - lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll - lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage - lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll - lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) - lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll - lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) - lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll - lib/net10.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId - lib/net462/Microsoft.Extensions.AI.Abstractions.dll - lib/net462/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage - lib/net462/Microsoft.Extensions.AI.Abstractions.dll - lib/net462/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) - lib/net462/Microsoft.Extensions.AI.Abstractions.dll - lib/net462/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) - lib/net462/Microsoft.Extensions.AI.Abstractions.dll - lib/net462/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId - lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll - lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage - lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll - lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) - lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll - lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) - lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll - lib/net8.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId - lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll - lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage - lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll - lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) - lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll - lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) - lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll - lib/net9.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.get_ModelId - lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.get_SpeechLanguage - lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.set_ModelId(System.String) - lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.AI.SpeechToTextOptions.set_SpeechLanguage(System.String) - lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.AI.Abstractions.dll - true - \ No newline at end of file diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/SpeechToText/SpeechToTextClientMetadata.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/SpeechToText/SpeechToTextClientMetadata.cs index e8fc8517ab1..24021577803 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/SpeechToText/SpeechToTextClientMetadata.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/SpeechToText/SpeechToTextClientMetadata.cs @@ -38,7 +38,7 @@ public SpeechToTextClientMetadata(string? providerName = null, Uri? providerUri /// Gets the ID of the default model used by this speech to text client. /// /// This value can be null if either the name is unknown or there are multiple possible models associated with this instance. - /// An individual request may override this value via . + /// An individual request may override this value via . /// public string? DefaultModelId { get; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/SpeechToText/SpeechToTextOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/SpeechToText/SpeechToTextOptions.cs index aacd4259db4..856442fbad3 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/SpeechToText/SpeechToTextOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/SpeechToText/SpeechToTextOptions.cs @@ -26,8 +26,9 @@ protected SpeechToTextOptions(SpeechToTextOptions? other) } AdditionalProperties = other.AdditionalProperties?.Clone(); - Transcription = other.Transcription; + ModelId = other.ModelId; RawRepresentationFactory = other.RawRepresentationFactory; + SpeechLanguage = other.SpeechLanguage; SpeechSampleRate = other.SpeechSampleRate; TextLanguage = other.TextLanguage; } @@ -35,8 +36,11 @@ protected SpeechToTextOptions(SpeechToTextOptions? other) /// Gets or sets any additional properties associated with the options. public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } - /// Gets or sets the transcription options for the speech to text request. - public TranscriptionOptions? Transcription { get; set; } + /// Gets or sets the model ID for the speech to text. + public string? ModelId { get; set; } + + /// Gets or sets the language of source speech. + public string? SpeechLanguage { get; set; } /// Gets or sets the sample rate of the speech input audio. public int? SpeechSampleRate { get; set; } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAISpeechToTextClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAISpeechToTextClient.cs index 2576b2d80f9..fbb8004f62b 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAISpeechToTextClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAISpeechToTextClient.cs @@ -134,7 +134,7 @@ public async IAsyncEnumerable GetStreamingTextAsync( { SpeechToTextResponseUpdate result = new() { - ModelId = options?.Transcription?.ModelId, + ModelId = options?.ModelId, RawRepresentation = update, }; @@ -165,14 +165,14 @@ void IDisposable.Dispose() private static bool IsTranslationRequest(SpeechToTextOptions? options) => options is not null && options.TextLanguage is not null && - (options.Transcription?.SpeechLanguage is null || options.Transcription.SpeechLanguage != options.TextLanguage); + (options.SpeechLanguage is null || options.SpeechLanguage != options.TextLanguage); /// Converts an extensions options instance to an OpenAI transcription options instance. private AudioTranscriptionOptions ToOpenAITranscriptionOptions(SpeechToTextOptions? options) { AudioTranscriptionOptions result = options?.RawRepresentationFactory?.Invoke(this) as AudioTranscriptionOptions ?? new(); - result.Language ??= options?.Transcription?.SpeechLanguage; + result.Language ??= options?.SpeechLanguage; return result; } diff --git a/src/Libraries/Microsoft.Extensions.AI/SpeechToText/OpenTelemetrySpeechToTextClient.cs b/src/Libraries/Microsoft.Extensions.AI/SpeechToText/OpenTelemetrySpeechToTextClient.cs index 58082cb165f..51d4b49af08 100644 --- a/src/Libraries/Microsoft.Extensions.AI/SpeechToText/OpenTelemetrySpeechToTextClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/SpeechToText/OpenTelemetrySpeechToTextClient.cs @@ -118,7 +118,7 @@ public override async Task GetTextAsync(Stream audioSpeech using Activity? activity = CreateAndConfigureActivity(options); Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; - string? requestModelId = options?.Transcription?.ModelId ?? _defaultModelId; + string? requestModelId = options?.ModelId ?? _defaultModelId; SpeechToTextResponse? response = null; Exception? error = null; @@ -146,7 +146,7 @@ public override async IAsyncEnumerable GetStreamingT using Activity? activity = CreateAndConfigureActivity(options); Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; - string? requestModelId = options?.Transcription?.ModelId ?? _defaultModelId; + string? requestModelId = options?.ModelId ?? _defaultModelId; IAsyncEnumerable updates; try @@ -201,7 +201,7 @@ public override async IAsyncEnumerable GetStreamingT Activity? activity = null; if (_activitySource.HasListeners()) { - string? modelId = options?.Transcription?.ModelId ?? _defaultModelId; + string? modelId = options?.ModelId ?? _defaultModelId; activity = _activitySource.StartActivity( string.IsNullOrWhiteSpace(modelId) ? OpenTelemetryConsts.GenAI.GenerateContentName : $"{OpenTelemetryConsts.GenAI.GenerateContentName} {modelId}", diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/SpeechToText/SpeechToTextOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/SpeechToText/SpeechToTextOptionsTests.cs index ec04bbbfffa..286d298f6af 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/SpeechToText/SpeechToTextOptionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/SpeechToText/SpeechToTextOptionsTests.cs @@ -13,12 +13,14 @@ public class SpeechToTextOptionsTests public void Constructor_Parameterless_PropsDefaulted() { SpeechToTextOptions options = new(); - Assert.Null(options.Transcription); + Assert.Null(options.ModelId); + Assert.Null(options.SpeechLanguage); Assert.Null(options.SpeechSampleRate); Assert.Null(options.AdditionalProperties); SpeechToTextOptions clone = options.Clone(); - Assert.Null(clone.Transcription); + Assert.Null(clone.ModelId); + Assert.Null(clone.SpeechLanguage); Assert.Null(clone.SpeechSampleRate); Assert.Null(clone.AdditionalProperties); } @@ -35,21 +37,21 @@ public void Properties_Roundtrip() Func rawRepresentationFactory = (c) => null; - var transcription = new TranscriptionOptions { ModelId = "modelId", SpeechLanguage = "en-US" }; - options.Transcription = transcription; + options.ModelId = "modelId"; + options.SpeechLanguage = "en-US"; options.SpeechSampleRate = 44100; options.AdditionalProperties = additionalProps; options.RawRepresentationFactory = rawRepresentationFactory; - Assert.Same(transcription, options.Transcription); - Assert.Equal("modelId", options.Transcription.ModelId); - Assert.Equal("en-US", options.Transcription.SpeechLanguage); + Assert.Equal("modelId", options.ModelId); + Assert.Equal("en-US", options.SpeechLanguage); Assert.Equal(44100, options.SpeechSampleRate); Assert.Same(additionalProps, options.AdditionalProperties); Assert.Same(rawRepresentationFactory, options.RawRepresentationFactory); SpeechToTextOptions clone = options.Clone(); - Assert.Same(transcription, clone.Transcription); + Assert.Equal("modelId", clone.ModelId); + Assert.Equal("en-US", clone.SpeechLanguage); Assert.Equal(44100, clone.SpeechSampleRate); Assert.Equal(additionalProps, clone.AdditionalProperties); Assert.Same(rawRepresentationFactory, clone.RawRepresentationFactory); @@ -65,7 +67,8 @@ public void JsonSerialization_Roundtrips() ["key"] = "value", }; - options.Transcription = new TranscriptionOptions { ModelId = "modelId", SpeechLanguage = "en-US" }; + options.ModelId = "modelId"; + options.SpeechLanguage = "en-US"; options.SpeechSampleRate = 44100; options.AdditionalProperties = additionalProps; @@ -74,9 +77,8 @@ public void JsonSerialization_Roundtrips() SpeechToTextOptions? deserialized = JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.SpeechToTextOptions); Assert.NotNull(deserialized); - Assert.NotNull(deserialized.Transcription); - Assert.Equal("modelId", deserialized.Transcription.ModelId); - Assert.Equal("en-US", deserialized.Transcription.SpeechLanguage); + Assert.Equal("modelId", deserialized.ModelId); + Assert.Equal("en-US", deserialized.SpeechLanguage); Assert.Equal(44100, deserialized.SpeechSampleRate); Assert.NotNull(deserialized.AdditionalProperties); @@ -91,14 +93,14 @@ public void CopyConstructors_EnableHierarchyCloning() { OptionsB b = new() { - Transcription = new TranscriptionOptions { ModelId = "test" }, + ModelId = "test", A = 42, B = 84, }; SpeechToTextOptions clone = b.Clone(); - Assert.Equal("test", clone.Transcription?.ModelId); + Assert.Equal("test", clone.ModelId); Assert.Equal(42, Assert.IsType(clone, exactMatch: false).A); Assert.Equal(84, Assert.IsType(clone, exactMatch: true).B); } @@ -157,10 +159,8 @@ public void JsonDeserialization_KnownPayload() { const string Json = """ { - "transcription": { - "modelId": "whisper-1", - "speechLanguage": "en-US" - }, + "modelId": "whisper-1", + "speechLanguage": "en-US", "speechSampleRate": 16000, "textLanguage": "en", "additionalProperties": { @@ -172,9 +172,8 @@ public void JsonDeserialization_KnownPayload() SpeechToTextOptions? result = JsonSerializer.Deserialize(Json, AIJsonUtilities.DefaultOptions); Assert.NotNull(result); - Assert.NotNull(result.Transcription); - Assert.Equal("whisper-1", result.Transcription.ModelId); - Assert.Equal("en-US", result.Transcription.SpeechLanguage); + Assert.Equal("whisper-1", result.ModelId); + Assert.Equal("en-US", result.SpeechLanguage); Assert.Equal(16000, result.SpeechSampleRate); Assert.Equal("en", result.TextLanguage); Assert.NotNull(result.AdditionalProperties); diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAISpeechToTextClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAISpeechToTextClientTests.cs index 4e6e7b0cd95..1e7aa1deb13 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAISpeechToTextClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAISpeechToTextClientTests.cs @@ -86,7 +86,7 @@ public async Task GetTextAsync_BasicRequestResponse(string? speechLanguage, stri using var audioSpeechStream = GetAudioStream(); var response = await client.GetTextAsync(audioSpeechStream, new SpeechToTextOptions { - Transcription = new TranscriptionOptions { SpeechLanguage = speechLanguage }, + SpeechLanguage = speechLanguage, TextLanguage = textLanguage }); @@ -161,7 +161,7 @@ public async Task GetStreamingTextAsync_BasicRequestResponse(string? speechLangu using var audioSpeechStream = GetAudioStream(); await foreach (var update in client.GetStreamingTextAsync(audioSpeechStream, new SpeechToTextOptions { - Transcription = new TranscriptionOptions { SpeechLanguage = speechLanguage }, + SpeechLanguage = speechLanguage, TextLanguage = textLanguage })) { @@ -196,7 +196,7 @@ public async Task GetStreamingTextAsync_BasicTranslateRequestResponse() using var audioSpeechStream = GetAudioStream(); await foreach (var update in client.GetStreamingTextAsync(audioSpeechStream, new SpeechToTextOptions { - Transcription = new TranscriptionOptions { SpeechLanguage = "pt" }, + SpeechLanguage = "pt", TextLanguage = textLanguage })) { @@ -233,7 +233,7 @@ public async Task GetTextAsync_Transcription_StronglyTypedOptions_AllSent() using var audioSpeechStream = GetAudioStream(); Assert.NotNull(await client.GetTextAsync(audioSpeechStream, new() { - Transcription = new TranscriptionOptions { SpeechLanguage = "en" }, + SpeechLanguage = "en", RawRepresentationFactory = (s) => new AudioTranscriptionOptions { diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/SpeechToText/ConfigureOptionsSpeechToTextClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/SpeechToText/ConfigureOptionsSpeechToTextClientTests.cs index c8b51fe9d47..6140b7ed354 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/SpeechToText/ConfigureOptionsSpeechToTextClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/SpeechToText/ConfigureOptionsSpeechToTextClientTests.cs @@ -33,7 +33,7 @@ public void ConfigureOptions_InvalidArgs_Throws() [InlineData(true)] public async Task ConfigureOptions_ReturnedInstancePassedToNextClient(bool nullProvidedOptions) { - SpeechToTextOptions? providedOptions = nullProvidedOptions ? null : new() { Transcription = new TranscriptionOptions { ModelId = "test" } }; + SpeechToTextOptions? providedOptions = nullProvidedOptions ? null : new() { ModelId = "test" }; SpeechToTextOptions? returnedOptions = null; SpeechToTextResponse expectedResponse = new([]); var expectedUpdates = Enumerable.Range(0, 3).Select(i => new SpeechToTextResponseUpdate()).ToArray(); @@ -63,11 +63,11 @@ public async Task ConfigureOptions_ReturnedInstancePassedToNextClient(bool nullP Assert.NotSame(providedOptions, options); if (nullProvidedOptions) { - Assert.Null(options.Transcription); + Assert.Null(options.ModelId); } else { - Assert.Equal(providedOptions!.Transcription?.ModelId, options.Transcription?.ModelId); + Assert.Equal(providedOptions!.ModelId, options.ModelId); } returnedOptions = options; diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/SpeechToText/LoggingSpeechToTextClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/SpeechToText/LoggingSpeechToTextClientTests.cs index 05b0654f3f8..79c09dd5c6f 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/SpeechToText/LoggingSpeechToTextClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/SpeechToText/LoggingSpeechToTextClientTests.cs @@ -70,7 +70,7 @@ public async Task GetTextAsync_LogsResponseInvocationAndCompletion(LogLevel leve using var audioSpeechStream = new MemoryStream(new byte[] { 1, 2, 3, 4 }); await client.GetTextAsync( audioSpeechStream, - new SpeechToTextOptions { Transcription = new TranscriptionOptions { SpeechLanguage = "pt" } }); + new SpeechToTextOptions { SpeechLanguage = "pt" }); var logs = collector.GetSnapshot(); if (level is LogLevel.Trace) @@ -120,7 +120,7 @@ static async IAsyncEnumerable GetUpdatesAsync() using var audioSpeechStream = new MemoryStream(new byte[] { 1, 2, 3, 4 }); await foreach (var update in client.GetStreamingTextAsync( audioSpeechStream, - new SpeechToTextOptions { Transcription = new TranscriptionOptions { SpeechLanguage = "pt" } })) + new SpeechToTextOptions { SpeechLanguage = "pt" })) { // nop } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/SpeechToText/OpenTelemetrySpeechToTextClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/SpeechToText/OpenTelemetrySpeechToTextClientTests.cs index 144912292c5..c243bf2bf12 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/SpeechToText/OpenTelemetrySpeechToTextClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/SpeechToText/OpenTelemetrySpeechToTextClientTests.cs @@ -91,7 +91,7 @@ static async IAsyncEnumerable TestClientStreamAsync( SpeechToTextOptions options = new() { - Transcription = new TranscriptionOptions { ModelId = "mycoolspeechmodel" }, + ModelId = "mycoolspeechmodel", AdditionalProperties = new() { ["service_tier"] = "value1", From d074823a6145c62798cc296ac417ed4983aacad7 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 10 Mar 2026 13:20:33 -0700 Subject: [PATCH 79/92] Use JsonInclude/Core pattern for experimental UsageDetails properties Apply the [JsonIgnore] public + [JsonInclude] internal Core backing property pattern to the 4 experimental audio/text token properties on UsageDetails, matching the convention used elsewhere (e.g. ChatOptions.AllowBackgroundResponses, ChatResponse.ContinuationToken). This enables JSON serialization of these properties when using the library's JsonSerializerOptions while keeping the public API gated behind [Experimental]. - InputAudioTokenCount -> InputAudioTokenCountCore - InputTextTokenCount -> InputTextTokenCountCore - OutputAudioTokenCount -> OutputAudioTokenCountCore - OutputTextTokenCount -> OutputTextTokenCountCore - Add tests for property roundtrip, Add summation, and JSON serialization/deserialization of the new properties --- .../UsageDetails.cs | 40 ++++++++++++++-- .../UsageDetailsTests.cs | 46 +++++++++++++++++++ 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/UsageDetails.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/UsageDetails.cs index 4af1aa83b6a..cf282e1a011 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/UsageDetails.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/UsageDetails.cs @@ -47,7 +47,15 @@ public class UsageDetails /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] [JsonIgnore] - public long? InputAudioTokenCount { get; set; } + public long? InputAudioTokenCount + { + get => InputAudioTokenCountCore; + set => InputAudioTokenCountCore = value; + } + + [JsonInclude] + [JsonPropertyName("inputAudioTokenCount")] + internal long? InputAudioTokenCountCore { get; set; } /// Gets or sets the number of text input tokens used. /// @@ -55,7 +63,15 @@ public class UsageDetails /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] [JsonIgnore] - public long? InputTextTokenCount { get; set; } + public long? InputTextTokenCount + { + get => InputTextTokenCountCore; + set => InputTextTokenCountCore = value; + } + + [JsonInclude] + [JsonPropertyName("inputTextTokenCount")] + internal long? InputTextTokenCountCore { get; set; } /// Gets or sets the number of audio output tokens used. /// @@ -63,7 +79,15 @@ public class UsageDetails /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] [JsonIgnore] - public long? OutputAudioTokenCount { get; set; } + public long? OutputAudioTokenCount + { + get => OutputAudioTokenCountCore; + set => OutputAudioTokenCountCore = value; + } + + [JsonInclude] + [JsonPropertyName("outputAudioTokenCount")] + internal long? OutputAudioTokenCountCore { get; set; } /// Gets or sets the number of text output tokens used. /// @@ -71,7 +95,15 @@ public class UsageDetails /// [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] [JsonIgnore] - public long? OutputTextTokenCount { get; set; } + public long? OutputTextTokenCount + { + get => OutputTextTokenCountCore; + set => OutputTextTokenCountCore = value; + } + + [JsonInclude] + [JsonPropertyName("outputTextTokenCount")] + internal long? OutputTextTokenCountCore { get; set; } /// Gets or sets a dictionary of additional usage counts. /// diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/UsageDetailsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/UsageDetailsTests.cs index e401a341b43..c4fdaccf312 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/UsageDetailsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/UsageDetailsTests.cs @@ -5,6 +5,8 @@ using System.Text.Json; using Xunit; +#pragma warning disable MEAI001 // Experimental API + namespace Microsoft.Extensions.AI; public class UsageDetailsTests @@ -18,6 +20,10 @@ public void Constructor_PropsDefault() Assert.Null(details.TotalTokenCount); Assert.Null(details.CachedInputTokenCount); Assert.Null(details.ReasoningTokenCount); + Assert.Null(details.InputAudioTokenCount); + Assert.Null(details.InputTextTokenCount); + Assert.Null(details.OutputAudioTokenCount); + Assert.Null(details.OutputTextTokenCount); Assert.Null(details.AdditionalCounts); } @@ -31,6 +37,10 @@ public void Properties_Roundtrip() TotalTokenCount = 30, CachedInputTokenCount = 5, ReasoningTokenCount = 8, + InputAudioTokenCount = 50, + InputTextTokenCount = 60, + OutputAudioTokenCount = 70, + OutputTextTokenCount = 80, AdditionalCounts = new() { ["custom"] = 100 } }; @@ -39,6 +49,10 @@ public void Properties_Roundtrip() Assert.Equal(30, details.TotalTokenCount); Assert.Equal(5, details.CachedInputTokenCount); Assert.Equal(8, details.ReasoningTokenCount); + Assert.Equal(50, details.InputAudioTokenCount); + Assert.Equal(60, details.InputTextTokenCount); + Assert.Equal(70, details.OutputAudioTokenCount); + Assert.Equal(80, details.OutputTextTokenCount); Assert.NotNull(details.AdditionalCounts); Assert.Equal(100, details.AdditionalCounts["custom"]); } @@ -60,6 +74,10 @@ public void Add_SumsAllProperties() TotalTokenCount = 30, CachedInputTokenCount = 5, ReasoningTokenCount = 8, + InputAudioTokenCount = 10, + InputTextTokenCount = 20, + OutputAudioTokenCount = 30, + OutputTextTokenCount = 40, }; UsageDetails details2 = new() @@ -69,6 +87,10 @@ public void Add_SumsAllProperties() TotalTokenCount = 40, CachedInputTokenCount = 7, ReasoningTokenCount = 12, + InputAudioTokenCount = 15, + InputTextTokenCount = 25, + OutputAudioTokenCount = 35, + OutputTextTokenCount = 45, }; details1.Add(details2); @@ -78,6 +100,10 @@ public void Add_SumsAllProperties() Assert.Equal(70, details1.TotalTokenCount); Assert.Equal(12, details1.CachedInputTokenCount); Assert.Equal(20, details1.ReasoningTokenCount); + Assert.Equal(25, details1.InputAudioTokenCount); + Assert.Equal(45, details1.InputTextTokenCount); + Assert.Equal(65, details1.OutputAudioTokenCount); + Assert.Equal(85, details1.OutputTextTokenCount); } [Fact] @@ -152,6 +178,10 @@ public void Serialization_Roundtrips() TotalTokenCount = 30, CachedInputTokenCount = 5, ReasoningTokenCount = 8, + InputAudioTokenCount = 50, + InputTextTokenCount = 60, + OutputAudioTokenCount = 70, + OutputTextTokenCount = 80, AdditionalCounts = new() { ["custom"] = 100 } }; @@ -164,6 +194,10 @@ public void Serialization_Roundtrips() Assert.Equal(details.TotalTokenCount, deserialized.TotalTokenCount); Assert.Equal(details.CachedInputTokenCount, deserialized.CachedInputTokenCount); Assert.Equal(details.ReasoningTokenCount, deserialized.ReasoningTokenCount); + Assert.Equal(details.InputAudioTokenCount, deserialized.InputAudioTokenCount); + Assert.Equal(details.InputTextTokenCount, deserialized.InputTextTokenCount); + Assert.Equal(details.OutputAudioTokenCount, deserialized.OutputAudioTokenCount); + Assert.Equal(details.OutputTextTokenCount, deserialized.OutputTextTokenCount); Assert.NotNull(deserialized.AdditionalCounts); Assert.Equal(100, deserialized.AdditionalCounts["custom"]); } @@ -186,6 +220,10 @@ public void Serialization_WithNullProperties_Roundtrips() Assert.Null(deserialized.TotalTokenCount); Assert.Null(deserialized.CachedInputTokenCount); Assert.Null(deserialized.ReasoningTokenCount); + Assert.Null(deserialized.InputAudioTokenCount); + Assert.Null(deserialized.InputTextTokenCount); + Assert.Null(deserialized.OutputAudioTokenCount); + Assert.Null(deserialized.OutputTextTokenCount); } [Fact] @@ -198,6 +236,10 @@ public void JsonDeserialization_KnownPayload() "totalTokenCount": 30, "cachedInputTokenCount": 5, "reasoningTokenCount": 8, + "inputAudioTokenCount": 50, + "inputTextTokenCount": 60, + "outputAudioTokenCount": 70, + "outputTextTokenCount": 80, "additionalCounts": { "custom": 100 } @@ -212,6 +254,10 @@ public void JsonDeserialization_KnownPayload() Assert.Equal(30, result.TotalTokenCount); Assert.Equal(5, result.CachedInputTokenCount); Assert.Equal(8, result.ReasoningTokenCount); + Assert.Equal(50, result.InputAudioTokenCount); + Assert.Equal(60, result.InputTextTokenCount); + Assert.Equal(70, result.OutputAudioTokenCount); + Assert.Equal(80, result.OutputTextTokenCount); Assert.NotNull(result.AdditionalCounts); Assert.Single(result.AdditionalCounts); Assert.Equal(100, result.AdditionalCounts["custom"]); From ded7128cf07115a91c77d00bfd1326bd8b19f6e9 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 10 Mar 2026 14:13:49 -0700 Subject: [PATCH 80/92] Fix SendAsync to propagate exceptions instead of swallowing them Replace silent return on cancellation with ThrowIfCancellationRequested, throw InvalidOperationException when session is not connected, and remove catch block that swallowed OperationCanceledException, ObjectDisposedException, and WebSocketException. Update tests to verify the new exception behavior. --- .../OpenAIRealtimeClientSession.cs | 68 +++++++++---------- .../OpenAIRealtimeClientSessionTests.cs | 16 +++-- 2 files changed, 43 insertions(+), 41 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs index dbccaf06bb1..aed05681c36 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs @@ -5,7 +5,6 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; -using System.Net.WebSockets; using System.Runtime.CompilerServices; using System.Text.Json; using System.Text.Json.Nodes; @@ -111,52 +110,47 @@ public async Task SendAsync(RealtimeClientMessage message, CancellationToken can { _ = Throw.IfNull(message); - if (cancellationToken.IsCancellationRequested || _sessionClient is null) + cancellationToken.ThrowIfCancellationRequested(); + + if (_sessionClient is null) { - return; + Throw.InvalidOperationException("The session is not connected."); } - try + switch (message) { - switch (message) - { - case SessionUpdateRealtimeClientMessage sessionUpdate: - await UpdateSessionAsync(sessionUpdate.Options, cancellationToken).ConfigureAwait(false); - break; + case SessionUpdateRealtimeClientMessage sessionUpdate: + await UpdateSessionAsync(sessionUpdate.Options, cancellationToken).ConfigureAwait(false); + break; - case CreateResponseRealtimeClientMessage responseCreate: - await SendResponseCreateAsync(responseCreate, cancellationToken).ConfigureAwait(false); - break; + case CreateResponseRealtimeClientMessage responseCreate: + await SendResponseCreateAsync(responseCreate, cancellationToken).ConfigureAwait(false); + break; - case CreateConversationItemRealtimeClientMessage itemCreate: - await SendConversationItemCreateAsync(itemCreate, cancellationToken).ConfigureAwait(false); - break; + case CreateConversationItemRealtimeClientMessage itemCreate: + await SendConversationItemCreateAsync(itemCreate, cancellationToken).ConfigureAwait(false); + break; - case InputAudioBufferAppendRealtimeClientMessage audioAppend: - await SendInputAudioAppendAsync(audioAppend, cancellationToken).ConfigureAwait(false); - break; + case InputAudioBufferAppendRealtimeClientMessage audioAppend: + await SendInputAudioAppendAsync(audioAppend, cancellationToken).ConfigureAwait(false); + break; - case InputAudioBufferCommitRealtimeClientMessage: - if (message.MessageId is not null) - { - var cmd = new Sdk.RealtimeClientCommandInputAudioBufferCommit { EventId = message.MessageId }; - await _sessionClient.SendCommandAsync(cmd, cancellationToken).ConfigureAwait(false); - } - else - { - await _sessionClient.CommitPendingAudioAsync(cancellationToken).ConfigureAwait(false); - } + case InputAudioBufferCommitRealtimeClientMessage: + if (message.MessageId is not null) + { + var cmd = new Sdk.RealtimeClientCommandInputAudioBufferCommit { EventId = message.MessageId }; + await _sessionClient.SendCommandAsync(cmd, cancellationToken).ConfigureAwait(false); + } + else + { + await _sessionClient.CommitPendingAudioAsync(cancellationToken).ConfigureAwait(false); + } - break; + break; - default: - await SendRawCommandAsync(message, cancellationToken).ConfigureAwait(false); - break; - } - } - catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException or WebSocketException) - { - // Expected during session teardown or cancellation. + default: + await SendRawCommandAsync(message, cancellationToken).ConfigureAwait(false); + break; } } diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientSessionTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientSessionTests.cs index c98a99e8d16..71faf895d93 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientSessionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeClientSessionTests.cs @@ -62,15 +62,23 @@ public async Task SendAsync_NullMessage_Throws() } [Fact] - public async Task SendAsync_CancelledToken_ReturnsSilently() + public async Task SendAsync_CancelledToken_ThrowsOperationCanceledException() { await using var session = new OpenAIRealtimeClientSession("key", "model"); using var cts = new CancellationTokenSource(); cts.Cancel(); - // Should not throw when cancellation is requested. - await session.SendAsync(new RealtimeClientMessage(), cts.Token); - Assert.Null(session.Options); + // Should throw when cancellation is requested. + await Assert.ThrowsAsync(() => session.SendAsync(new RealtimeClientMessage(), cts.Token)); + } + + [Fact] + public async Task SendAsync_NotConnected_ThrowsInvalidOperationException() + { + await using var session = new OpenAIRealtimeClientSession("key", "model"); + + // Should throw when session is not connected. + await Assert.ThrowsAsync(() => session.SendAsync(new RealtimeClientMessage())); } [Fact] From 2c7a1cea56a979129baa4e09301fcd939f2565e1 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 10 Mar 2026 15:31:24 -0700 Subject: [PATCH 81/92] Replace toolMap dictionary with FindTool delegate pattern Refactor FunctionInvocationProcessor to accept a Func delegate instead of Dictionary? to align with main branch's FindTool pattern and avoid reintroducing the removed toolMap. --- .../FunctionInvokingChatClient.cs | 22 ++------- .../Common/FunctionInvocationHelpers.cs | 35 -------------- .../Common/FunctionInvocationProcessor.cs | 14 +++--- .../FunctionInvokingRealtimeClientSession.cs | 46 +++++++++++++++---- 4 files changed, 50 insertions(+), 67 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs index 70fdf7e4912..022d0ad53e9 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs @@ -296,7 +296,6 @@ public override async Task GetResponseAsync( List? functionCallContents = null; // function call contents that need responding to in the current turn bool lastIterationHadConversationId = false; // whether the last iteration's response had a ConversationId set int consecutiveErrorCount = 0; - Dictionary? toolMap = null; bool anyToolsRequireApproval = false; if (HasAnyApprovalContent(originalMessages)) @@ -404,12 +403,9 @@ public override async Task GetResponseAsync( // Prepare the history for the next iteration. FixupHistories(originalMessages, ref messages, ref augmentedHistory, response, responseMessages, ref lastIterationHadConversationId); - // Recompute toolMap on each iteration to respect ChatOptions.Tools modifications by functions. - (toolMap, _) = FunctionInvocationHelpers.CreateToolsMap(AdditionalTools, options?.Tools); - // Add the responses from the function calls into the augmented history and also into the tracked // list of response messages. - var modeAndMessages = await ProcessFunctionCallsAsync(augmentedHistory, options, toolMap, functionCallContents!, iteration, consecutiveErrorCount, isStreaming: false, cancellationToken); + var modeAndMessages = await ProcessFunctionCallsAsync(augmentedHistory, options, functionCallContents!, iteration, consecutiveErrorCount, isStreaming: false, cancellationToken); responseMessages.AddRange(modeAndMessages.MessagesAdded); consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; @@ -453,7 +449,6 @@ public override async IAsyncEnumerable GetStreamingResponseA bool lastIterationHadConversationId = false; // whether the last iteration's response had a ConversationId set List updates = []; // updates from the current response int consecutiveErrorCount = 0; - Dictionary? toolMap = null; bool anyToolsRequireApproval = false; // This is a synthetic ID since we're generating the tool messages instead of getting them from @@ -674,11 +669,8 @@ public override async IAsyncEnumerable GetStreamingResponseA // Prepare the history for the next iteration. FixupHistories(originalMessages, ref messages, ref augmentedHistory, response, responseMessages, ref lastIterationHadConversationId); - // Recompute toolMap on each iteration to respect ChatOptions.Tools modifications by functions. - (toolMap, _) = FunctionInvocationHelpers.CreateToolsMap(AdditionalTools, options?.Tools); - // Process all of the functions, adding their results into the history. - var modeAndMessages = await ProcessFunctionCallsAsync(augmentedHistory, options, toolMap, functionCallContents!, iteration, consecutiveErrorCount, isStreaming: true, cancellationToken); + var modeAndMessages = await ProcessFunctionCallsAsync(augmentedHistory, options, functionCallContents!, iteration, consecutiveErrorCount, isStreaming: true, cancellationToken); responseMessages.AddRange(modeAndMessages.MessagesAdded); consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; @@ -1132,7 +1124,6 @@ private bool ShouldTerminateLoopBasedOnHandleableFunctions(List /// The current chat contents, inclusive of the function call contents being processed. /// The options used for the response being processed. - /// Map from tool name to available AI tools. /// The function call contents representing the functions to be invoked. /// The iteration number of how many roundtrips have been made to the inner client. /// The number of consecutive iterations, prior to this one, that were recorded as having function invocation errors. @@ -1140,7 +1131,7 @@ private bool ShouldTerminateLoopBasedOnHandleableFunctions(ListThe to monitor for cancellation requests. /// A value indicating how the caller should proceed. private async Task<(bool ShouldTerminate, int NewConsecutiveErrorCount, IList MessagesAdded)> ProcessFunctionCallsAsync( - List messages, ChatOptions? options, Dictionary? toolMap, + List messages, ChatOptions? options, List functionCallContents, int iteration, int consecutiveErrorCount, bool isStreaming, CancellationToken cancellationToken) { @@ -1153,7 +1144,7 @@ private bool ShouldTerminateLoopBasedOnHandleableFunctions(List FindTool(name, options?.Tools, AdditionalTools), AllowConcurrentInvocation, (callContent, aiFunction, callIndex) => new FunctionInvocationContext { @@ -1712,12 +1703,9 @@ private IList ReplaceFunctionCallsWithApprovalRequests( // Check if there are any function calls to do for any approved functions and execute them. if (notInvokedApprovals is { Count: > 0 }) { - // Compute the tool map for this invocation - var (toolMap, _) = FunctionInvocationHelpers.CreateToolsMap(AdditionalTools, options?.Tools); - // The FRC that is generated here is already added to originalMessages by ProcessFunctionCallsAsync. var modeAndMessages = await ProcessFunctionCallsAsync( - originalMessages, options, toolMap, notInvokedApprovals.Select(x => x.Response.ToolCall).OfType().ToList(), 0, consecutiveErrorCount, isStreaming, cancellationToken); + originalMessages, options, notInvokedApprovals.Select(x => x.Response.ToolCall).OfType().ToList(), 0, consecutiveErrorCount, isStreaming, cancellationToken); consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; return (modeAndMessages.MessagesAdded, modeAndMessages.ShouldTerminate, consecutiveErrorCount); diff --git a/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationHelpers.cs b/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationHelpers.cs index de2b3cc745c..d9f97d3e8c4 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationHelpers.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationHelpers.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.Collections.Generic; using System.Diagnostics; namespace Microsoft.Extensions.AI; @@ -31,38 +30,4 @@ internal static TimeSpan GetElapsedTime(long startingTimestamp) => #else new((long)((Stopwatch.GetTimestamp() - startingTimestamp) * ((double)TimeSpan.TicksPerSecond / Stopwatch.Frequency))); #endif - - /// Creates a mapping from tool names to the corresponding tools. - /// - /// The lists of tools to combine into a single dictionary. Only - /// instances are included. Tools from later lists take precedence over tools from earlier lists - /// if they have the same name. - /// - /// A tuple containing the tool map and a flag indicating whether any tools require approval. - internal static (Dictionary? ToolMap, bool AnyRequireApproval) CreateToolsMap(params ReadOnlySpan?> toolLists) - { - Dictionary? map = null; - bool anyRequireApproval = false; - - foreach (var toolList in toolLists) - { - if (toolList?.Count is int count && count > 0) - { - for (int i = 0; i < count; i++) - { - AITool tool = toolList[i]; - if (tool is AIFunctionDeclaration) - { - anyRequireApproval |= tool.GetService() is not null; - - // Later lists take precedence (options?.Tools overrides AdditionalTools) - map ??= new(StringComparer.Ordinal); - map[tool.Name] = tool; - } - } - } - } - - return (map, anyRequireApproval); - } } diff --git a/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationProcessor.cs b/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationProcessor.cs index 19b472a5e5d..c4a1d2448e4 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationProcessor.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationProcessor.cs @@ -52,7 +52,7 @@ public FunctionInvocationProcessor( /// Processes multiple function calls, either concurrently or serially. /// /// The function calls to process. - /// Map from tool name to tool. + /// Delegate to look up a tool by name. Returns null if not found. /// Whether to allow concurrent invocation. /// Delegate to create a for each function call. /// Delegate to set the current context (for AsyncLocal flow). @@ -61,7 +61,7 @@ public FunctionInvocationProcessor( /// A list of function invocation results. public async Task> ProcessFunctionCallsAsync( List functionCallContents, - Dictionary? toolMap, + Func findTool, bool allowConcurrentInvocation, Func createContext, Action setCurrentContext, @@ -76,7 +76,7 @@ public async Task> ProcessFunctionCallsAsync( results.AddRange(await Task.WhenAll( from callIndex in Enumerable.Range(0, functionCallContents.Count) select ProcessSingleFunctionCallAsync( - functionCallContents[callIndex], toolMap, callIndex, + functionCallContents[callIndex], findTool, callIndex, createContext, setCurrentContext, captureExceptions: true, cancellationToken)).ConfigureAwait(false)); } else @@ -85,7 +85,7 @@ select ProcessSingleFunctionCallAsync( for (int callIndex = 0; callIndex < functionCallContents.Count; callIndex++) { var result = await ProcessSingleFunctionCallAsync( - functionCallContents[callIndex], toolMap, callIndex, + functionCallContents[callIndex], findTool, callIndex, createContext, setCurrentContext, captureExceptionsWhenSerial, cancellationToken).ConfigureAwait(false); results.Add(result); @@ -105,7 +105,7 @@ select ProcessSingleFunctionCallAsync( /// private async Task ProcessSingleFunctionCallAsync( FunctionCallContent callContent, - Dictionary? toolMap, + Func findTool, int callIndex, Func createContext, Action setCurrentContext, @@ -113,8 +113,8 @@ private async Task ProcessSingleFunctionCallAsync( CancellationToken cancellationToken) { // Look up the AIFunction for the function call. If the requested function isn't available, send back an error. - if (toolMap is null || - !toolMap.TryGetValue(callContent.Name, out AITool? tool)) + AITool? tool = findTool(callContent.Name); + if (tool is null) { FunctionInvocationLogger.LogFunctionNotFound(_logger, callContent.Name); return new(terminate: false, FunctionInvocationStatus.NotFound, callContent, result: null, exception: null); diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs index 2f54677b8c4..4022b3e71e2 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs @@ -223,6 +223,40 @@ private static bool ExtractFunctionCalls(ResponseOutputItemRealtimeServerMessage return functionCallContents.Count > 0; } + /// Finds a tool by name in the specified tool lists. + private static AIFunctionDeclaration? FindTool(string name, params ReadOnlySpan?> toolLists) + { + foreach (var toolList in toolLists) + { + if (toolList is not null) + { + foreach (AITool tool in toolList) + { + if (tool is AIFunctionDeclaration declaration && string.Equals(tool.Name, name, StringComparison.Ordinal)) + { + return declaration; + } + } + } + } + + return null; + } + + /// Checks whether there are any tools in the specified tool lists. + private static bool HasAnyTools(params ReadOnlySpan?> toolLists) + { + foreach (var toolList in toolLists) + { + if (toolList?.Count > 0) + { + return true; + } + } + + return false; + } + /// Gets whether the function calling loop should exit based on the function call requests. /// /// This mirrors the logic in FunctionInvokingChatClient.ShouldTerminateLoopBasedOnHandleableFunctions. @@ -232,9 +266,7 @@ private static bool ExtractFunctionCalls(ResponseOutputItemRealtimeServerMessage /// private bool ShouldTerminateBasedOnFunctionCalls(List functionCallContents) { - var (toolMap, _) = FunctionInvocationHelpers.CreateToolsMap(AdditionalTools, _innerSession.Options?.Tools as IList); - - if (toolMap is null || toolMap.Count == 0) + if (!HasAnyTools(AdditionalTools, _innerSession.Options?.Tools as IList)) { // No tools available at all. If TerminateOnUnknownCalls, stop the loop. if (TerminateOnUnknownCalls) @@ -252,7 +284,8 @@ private bool ShouldTerminateBasedOnFunctionCalls(List funct foreach (var fcc in functionCallContents) { - if (toolMap.TryGetValue(fcc.Name, out AITool? tool)) + AIFunctionDeclaration? tool = FindTool(fcc.Name, AdditionalTools, _innerSession.Options?.Tools as IList); + if (tool is not null) { if (tool is not AIFunction) { @@ -279,15 +312,12 @@ private bool ShouldTerminateBasedOnFunctionCalls(List funct int consecutiveErrorCount, CancellationToken cancellationToken) { - // Compute toolMap to ensure we always use the latest tools - var (toolMap, _) = FunctionInvocationHelpers.CreateToolsMap(AdditionalTools, _innerSession.Options?.Tools as IList); - var captureCurrentIterationExceptions = consecutiveErrorCount < MaximumConsecutiveErrorsPerRequest; // Use the processor to handle function calls var results = await Processor.ProcessFunctionCallsAsync( functionCallContents, - toolMap, + name => FindTool(name, AdditionalTools, _innerSession.Options?.Tools as IList), AllowConcurrentInvocation, (callContent, aiFunction, _) => new FunctionInvocationContext { From 5c6a8c4012b44eccb35652452836158407086099 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 10 Mar 2026 16:04:50 -0700 Subject: [PATCH 82/92] Use IEnumerable in FindTool/HasAnyTools to avoid unsafe IList cast Options.Tools is IReadOnlyList which does not implement IList, so the 'as IList' cast could silently return null and ignore tools. Use IEnumerable which both IList and IReadOnlyList implement. --- .../FunctionInvokingRealtimeClientSession.cs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs index 4022b3e71e2..1750cbb1ee2 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs @@ -224,7 +224,7 @@ private static bool ExtractFunctionCalls(ResponseOutputItemRealtimeServerMessage } /// Finds a tool by name in the specified tool lists. - private static AIFunctionDeclaration? FindTool(string name, params ReadOnlySpan?> toolLists) + private static AIFunctionDeclaration? FindTool(string name, params ReadOnlySpan?> toolLists) { foreach (var toolList in toolLists) { @@ -244,13 +244,17 @@ private static bool ExtractFunctionCalls(ResponseOutputItemRealtimeServerMessage } /// Checks whether there are any tools in the specified tool lists. - private static bool HasAnyTools(params ReadOnlySpan?> toolLists) + private static bool HasAnyTools(params ReadOnlySpan?> toolLists) { foreach (var toolList in toolLists) { - if (toolList?.Count > 0) + if (toolList is not null) { - return true; + using var enumerator = toolList.GetEnumerator(); + if (enumerator.MoveNext()) + { + return true; + } } } @@ -266,7 +270,7 @@ private static bool HasAnyTools(params ReadOnlySpan?> toolLists) /// private bool ShouldTerminateBasedOnFunctionCalls(List functionCallContents) { - if (!HasAnyTools(AdditionalTools, _innerSession.Options?.Tools as IList)) + if (!HasAnyTools(AdditionalTools, _innerSession.Options?.Tools)) { // No tools available at all. If TerminateOnUnknownCalls, stop the loop. if (TerminateOnUnknownCalls) @@ -284,7 +288,7 @@ private bool ShouldTerminateBasedOnFunctionCalls(List funct foreach (var fcc in functionCallContents) { - AIFunctionDeclaration? tool = FindTool(fcc.Name, AdditionalTools, _innerSession.Options?.Tools as IList); + AIFunctionDeclaration? tool = FindTool(fcc.Name, AdditionalTools, _innerSession.Options?.Tools); if (tool is not null) { if (tool is not AIFunction) @@ -317,7 +321,7 @@ private bool ShouldTerminateBasedOnFunctionCalls(List funct // Use the processor to handle function calls var results = await Processor.ProcessFunctionCallsAsync( functionCallContents, - name => FindTool(name, AdditionalTools, _innerSession.Options?.Tools as IList), + name => FindTool(name, AdditionalTools, _innerSession.Options?.Tools), AllowConcurrentInvocation, (callContent, aiFunction, _) => new FunctionInvocationContext { From 36dbe01671e96859d6e64f5b43dd3a625ef12150 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 10 Mar 2026 16:12:32 -0700 Subject: [PATCH 83/92] Document that function invocation blocks message processing loop Add known limitation note to FunctionInvokingRealtimeClientSession XML docs explaining that incoming server messages (including user interruptions) are buffered during function invocation. --- .../Realtime/FunctionInvokingRealtimeClientSession.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs index 1750cbb1ee2..7c79bc449fb 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs @@ -46,6 +46,11 @@ namespace Microsoft.Extensions.AI; /// (the default), multiple concurrent requests to this same instance and using the same tools could result in those /// tools being used concurrently (one per request). /// +/// +/// Known limitation: Function invocation blocks the message processing loop. While functions are being +/// invoked, incoming server messages (including user interruptions) are buffered and not processed until the +/// invocation completes. +/// /// internal sealed class FunctionInvokingRealtimeClientSession : IRealtimeClientSession { From 0a288dc38d957c904f9d7fa2d3901f7617938afc Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 10 Mar 2026 16:45:09 -0700 Subject: [PATCH 84/92] Add distinct ConversationItem message types to prevent double invocation ConversationItemAdded and ConversationItemDone were incorrectly mapped to ResponseOutputItemAdded/Done, which could cause the function invoking session to invoke the same function call twice. --- .../Realtime/RealtimeServerMessageType.cs | 6 ++++++ .../OpenAIRealtimeClientSession.cs | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs index d7ceb0d52ca..298b2e03655 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs @@ -84,6 +84,12 @@ namespace Microsoft.Extensions.AI; /// Gets a message type indicating an individual output item has been added to the response. public static RealtimeServerMessageType ResponseOutputItemAdded { get; } = new("ResponseOutputItemAdded"); + /// Gets a message type indicating a conversation item has been added. + public static RealtimeServerMessageType ConversationItemAdded { get; } = new("ConversationItemAdded"); + + /// Gets a message type indicating a conversation item is complete. + public static RealtimeServerMessageType ConversationItemDone { get; } = new("ConversationItemDone"); + /// Gets a message type indicating an error occurred while processing the request. public static RealtimeServerMessageType Error { get; } = new("Error"); diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs index aed05681c36..f30fdd6581f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs @@ -758,8 +758,8 @@ private static BinaryData ExtractAudioBinaryData(DataContent content) Sdk.RealtimeServerUpdateConversationItemInputAudioTranscriptionDelta e => MapInputTranscriptionDelta(e), Sdk.RealtimeServerUpdateConversationItemInputAudioTranscriptionCompleted e => MapInputTranscriptionCompleted(e), Sdk.RealtimeServerUpdateConversationItemInputAudioTranscriptionFailed e => MapInputTranscriptionFailed(e), - Sdk.RealtimeServerUpdateConversationItemAdded e => MapConversationItem(e.EventId, e.Item, RealtimeServerMessageType.ResponseOutputItemAdded, e), - Sdk.RealtimeServerUpdateConversationItemDone e => MapConversationItem(e.EventId, e.Item, RealtimeServerMessageType.ResponseOutputItemDone, e), + Sdk.RealtimeServerUpdateConversationItemAdded e => MapConversationItem(e.EventId, e.Item, RealtimeServerMessageType.ConversationItemAdded, e), + Sdk.RealtimeServerUpdateConversationItemDone e => MapConversationItem(e.EventId, e.Item, RealtimeServerMessageType.ConversationItemDone, e), Sdk.RealtimeServerUpdateResponseMcpCallInProgress e => MapMcpCallEvent(e.EventId, e.ItemId, e.OutputIndex, new RealtimeServerMessageType("McpCallInProgress"), e), Sdk.RealtimeServerUpdateResponseMcpCallCompleted e => MapMcpCallEvent(e.EventId, e.ItemId, e.OutputIndex, new RealtimeServerMessageType("McpCallCompleted"), e), Sdk.RealtimeServerUpdateResponseMcpCallFailed e => MapMcpCallEvent(e.EventId, e.ItemId, e.OutputIndex, new RealtimeServerMessageType("McpCallFailed"), e), From 75734cd7cfddbbb444402569cc69ce170e00605e Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 10 Mar 2026 16:54:12 -0700 Subject: [PATCH 85/92] Remove custom ToolChoice telemetry attribute Remove gen_ai.request.tool_choice custom attribute that is not part of the OpenTelemetry GenAI semantic conventions. --- .../OpenTelemetryConsts.cs | 8 - .../OpenTelemetryRealtimeClientSession.cs | 18 -- .../OpenTelemetryRealtimeClientTests.cs | 177 ------------------ 3 files changed, 203 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs index 4fdee56ab9a..67a99febe8d 100644 --- a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs +++ b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs @@ -129,14 +129,6 @@ public static class Request public const string Temperature = "gen_ai.request.temperature"; public const string TopK = "gen_ai.request.top_k"; public const string TopP = "gen_ai.request.top_p"; - - /// - /// The tool choice mode for the request. - /// This is a custom attribute NOT part of the OpenTelemetry GenAI semantic conventions (as of v1.39). - /// Values: "none", "auto", "required", or a specific tool name when a tool is forced. - /// Custom attribute: "gen_ai.request.tool_choice". - /// - public const string ToolChoice = "gen_ai.request.tool_choice"; } public static class Response diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs index 1f87a47e110..09586c4d218 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs @@ -754,24 +754,6 @@ private static string SerializeMessages(IEnumerable message }), RealtimeOtelContext.Default.IEnumerableRealtimeOtelFunction)); } } - - // Tool choice mode (custom attribute - not part of OTel GenAI spec) - string? toolChoice = null; - if (options.ToolMode is { } toolMode) - { - toolChoice = toolMode switch - { - RequiredChatToolMode r when r.RequiredFunctionName is not null => r.RequiredFunctionName, - RequiredChatToolMode => "required", - NoneChatToolMode => "none", - _ => "auto", - }; - } - - if (toolChoice is not null) - { - _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.ToolChoice, toolChoice); - } } } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientTests.cs index 13397e99eed..16e5d911890 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/OpenTelemetryRealtimeClientTests.cs @@ -413,183 +413,6 @@ static async IAsyncEnumerable TranscriptionCallbackAsync( Assert.Equal("transcription", activity.GetTagItem("gen_ai.realtime.session_kind")); } - [Theory] - [InlineData("none", "none")] - [InlineData("auto", "auto")] - [InlineData("required", "required")] - public async Task ToolChoiceMode_Logged(string modeKey, string expectedValue) - { - ChatToolMode mode = modeKey switch - { - "none" => ChatToolMode.None, - "auto" => ChatToolMode.Auto, - "required" => ChatToolMode.RequireAny, - _ => throw new ArgumentException(modeKey), - }; - - var sourceName = Guid.NewGuid().ToString(); - var activities = new List(); - using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() - .AddSource(sourceName) - .AddInMemoryExporter(activities) - .Build(); - - await using var innerSession = new TestRealtimeClientSession - { - Options = new RealtimeSessionOptions - { - Model = "test-model", - ToolMode = mode, - Tools = [AIFunctionFactory.Create((string query) => query, "Search")], - }, - GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), - }; - - using var innerClient = new TestRealtimeClient(innerSession); - using var client = innerClient - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName) - .Build(); - await using var session = await client.CreateSessionAsync(); - - await foreach (var msg in GetClientMessagesAsync()) - { - await session.SendAsync(msg); - } - - await foreach (var response in session.GetStreamingResponseAsync()) - { - // Consume - } - - var activity = Assert.Single(activities); - Assert.Equal(expectedValue, activity.GetTagItem("gen_ai.request.tool_choice")); - } - - [Fact] - public async Task AIFunction_ForcedTool_Logged() - { - var sourceName = Guid.NewGuid().ToString(); - var activities = new List(); - using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() - .AddSource(sourceName) - .AddInMemoryExporter(activities) - .Build(); - - await using var innerSession = new TestRealtimeClientSession - { - Options = new RealtimeSessionOptions - { - Model = "test-model", - ToolMode = ChatToolMode.RequireSpecific("SpecificSearch"), - }, - GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), - }; - - using var innerClient = new TestRealtimeClient(innerSession); - using var client = innerClient - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName) - .Build(); - await using var session = await client.CreateSessionAsync(); - - await foreach (var msg in GetClientMessagesAsync()) - { - await session.SendAsync(msg); - } - - await foreach (var response in session.GetStreamingResponseAsync()) - { - // Consume - } - - var activity = Assert.Single(activities); - Assert.Equal("SpecificSearch", activity.GetTagItem("gen_ai.request.tool_choice")); - } - -#pragma warning disable MEAI001 // Type is for evaluation purposes only - [Fact] - public async Task RequireAny_ToolMode_Logged() - { - var sourceName = Guid.NewGuid().ToString(); - var activities = new List(); - using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() - .AddSource(sourceName) - .AddInMemoryExporter(activities) - .Build(); - - await using var innerSession = new TestRealtimeClientSession - { - Options = new RealtimeSessionOptions - { - Model = "test-model", - ToolMode = ChatToolMode.RequireAny, - }, - GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), - }; - - using var innerClient = new TestRealtimeClient(innerSession); - using var client = innerClient - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName) - .Build(); - await using var session = await client.CreateSessionAsync(); - - await foreach (var msg in GetClientMessagesAsync()) - { - await session.SendAsync(msg); - } - - await foreach (var response in session.GetStreamingResponseAsync()) - { - // Consume - } - - var activity = Assert.Single(activities); - Assert.Equal("required", activity.GetTagItem("gen_ai.request.tool_choice")); - } -#pragma warning restore MEAI001 - - [Fact] - public async Task NoToolChoice_NotLogged() - { - var sourceName = Guid.NewGuid().ToString(); - var activities = new List(); - using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() - .AddSource(sourceName) - .AddInMemoryExporter(activities) - .Build(); - - await using var innerSession = new TestRealtimeClientSession - { - Options = new RealtimeSessionOptions - { - Model = "test-model", - }, - GetStreamingResponseAsyncCallback = (cancellationToken) => SimpleCallbackAsync(cancellationToken), - }; - - using var innerClient = new TestRealtimeClient(innerSession); - using var client = innerClient - .AsBuilder() - .UseOpenTelemetry(sourceName: sourceName) - .Build(); - await using var session = await client.CreateSessionAsync(); - - await foreach (var msg in GetClientMessagesAsync()) - { - await session.SendAsync(msg); - } - - await foreach (var response in session.GetStreamingResponseAsync()) - { - // Consume - } - - var activity = Assert.Single(activities); - Assert.Null(activity.GetTagItem("gen_ai.request.tool_choice")); - } - [Fact] public async Task ToolCallContentInClientMessages_LoggedAsInputMessages() { From f6039abfd0302b86bcc23ec2d72f8922b07defd3 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 10 Mar 2026 17:06:22 -0700 Subject: [PATCH 86/92] Add null validation to Item property setter Consistent with constructor which already validates via Throw.IfNull. --- .../CreateConversationItemRealtimeClientMessage.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateConversationItemRealtimeClientMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateConversationItemRealtimeClientMessage.cs index 0f1f245e00c..29d1e7183db 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateConversationItemRealtimeClientMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateConversationItemRealtimeClientMessage.cs @@ -13,17 +13,23 @@ namespace Microsoft.Extensions.AI; [Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)] public class CreateConversationItemRealtimeClientMessage : RealtimeClientMessage { + private RealtimeConversationItem _item; + /// /// Initializes a new instance of the class. /// /// The conversation item to create. public CreateConversationItemRealtimeClientMessage(RealtimeConversationItem item) { - Item = Throw.IfNull(item); + _item = Throw.IfNull(item); } /// /// Gets or sets the conversation item to create. /// - public RealtimeConversationItem Item { get; set; } + public RealtimeConversationItem Item + { + get => _item; + set => _item = Throw.IfNull(value); + } } From 1f217b17e07ceb84a6f52bfafd327e1afa0d2224 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 10 Mar 2026 17:15:41 -0700 Subject: [PATCH 87/92] Add XML doc tags to realtime types Document ArgumentNullException on constructors and property setters that validate via Throw.IfNull/Throw.IfNullOrWhitespace. --- .../Realtime/CreateConversationItemRealtimeClientMessage.cs | 3 +++ .../Realtime/InputAudioBufferAppendRealtimeClientMessage.cs | 3 +++ .../Realtime/RealtimeServerMessageType.cs | 1 + .../Realtime/RealtimeSessionKind.cs | 1 + .../Realtime/SessionUpdateRealtimeClientMessage.cs | 2 ++ 5 files changed, 10 insertions(+) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateConversationItemRealtimeClientMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateConversationItemRealtimeClientMessage.cs index 29d1e7183db..5db5b3b91e7 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateConversationItemRealtimeClientMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/CreateConversationItemRealtimeClientMessage.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Diagnostics.CodeAnalysis; using Microsoft.Shared.DiagnosticIds; using Microsoft.Shared.Diagnostics; @@ -19,6 +20,7 @@ public class CreateConversationItemRealtimeClientMessage : RealtimeClientMessage /// Initializes a new instance of the class. /// /// The conversation item to create. + /// is . public CreateConversationItemRealtimeClientMessage(RealtimeConversationItem item) { _item = Throw.IfNull(item); @@ -27,6 +29,7 @@ public CreateConversationItemRealtimeClientMessage(RealtimeConversationItem item /// /// Gets or sets the conversation item to create. /// + /// is . public RealtimeConversationItem Item { get => _item; diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/InputAudioBufferAppendRealtimeClientMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/InputAudioBufferAppendRealtimeClientMessage.cs index 1f20903fb74..c255fc689cf 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/InputAudioBufferAppendRealtimeClientMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/InputAudioBufferAppendRealtimeClientMessage.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Diagnostics.CodeAnalysis; using Microsoft.Shared.DiagnosticIds; using Microsoft.Shared.Diagnostics; @@ -19,6 +20,7 @@ public class InputAudioBufferAppendRealtimeClientMessage : RealtimeClientMessage /// Initializes a new instance of the class. /// /// The data content containing the audio buffer data to append. + /// is . public InputAudioBufferAppendRealtimeClientMessage(DataContent audioContent) { _content = Throw.IfNull(audioContent); @@ -30,6 +32,7 @@ public InputAudioBufferAppendRealtimeClientMessage(DataContent audioContent) /// /// The content should include the audio buffer data that needs to be appended to the input audio buffer. /// + /// is . public DataContent Content { get => _content; diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs index 298b2e03655..e7c9f33c65c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeServerMessageType.cs @@ -102,6 +102,7 @@ namespace Microsoft.Extensions.AI; /// Initializes a new instance of the struct with the provided value. /// /// The value to associate with this . + /// is or whitespace. [JsonConstructor] public RealtimeServerMessageType(string value) { diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionKind.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionKind.cs index c612ac08c5a..7d79aeb68ed 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionKind.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionKind.cs @@ -39,6 +39,7 @@ namespace Microsoft.Extensions.AI; /// Initializes a new instance of the struct with the provided value. /// The value to associate with this . + /// is or whitespace. [JsonConstructor] public RealtimeSessionKind(string value) { diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SessionUpdateRealtimeClientMessage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SessionUpdateRealtimeClientMessage.cs index a2c1bbb614f..f07ec862dd4 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SessionUpdateRealtimeClientMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/SessionUpdateRealtimeClientMessage.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Diagnostics.CodeAnalysis; using Microsoft.Shared.DiagnosticIds; using Microsoft.Shared.Diagnostics; @@ -28,6 +29,7 @@ public class SessionUpdateRealtimeClientMessage : RealtimeClientMessage /// Initializes a new instance of the class. /// /// The session options to apply. + /// is . public SessionUpdateRealtimeClientMessage(RealtimeSessionOptions options) { Options = Throw.IfNull(options); From df1c338034ac973a1379935230eb467bb36ae5f8 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 10 Mar 2026 17:21:11 -0700 Subject: [PATCH 88/92] Use OrdinalIgnoreCase comparer for MCP headers dictionary HTTP headers are case-insensitive per RFC 7230. --- .../OpenAIRealtimeClientSession.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs index f30fdd6581f..9de69f9fcfa 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs @@ -517,7 +517,7 @@ private static Sdk.RealtimeMcpTool ToRealtimeMcpTool(HostedMcpServerTool mcpTool if (mcpTool.Headers is { } headers) { - var sdkHeaders = new Dictionary(); + var sdkHeaders = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var kvp in headers) { sdkHeaders[kvp.Key] = kvp.Value; From 73443bab24600da8c4fe1882b7c7c3cc8c196fdc Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 10 Mar 2026 17:30:48 -0700 Subject: [PATCH 89/92] Change RealtimeResponseStatus from const fields to static properties Avoids baking values into consuming assemblies at compile time. --- .../Realtime/RealtimeResponseStatus.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeResponseStatus.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeResponseStatus.cs index 133bcabec79..f89fcd4224c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeResponseStatus.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeResponseStatus.cs @@ -19,24 +19,24 @@ namespace Microsoft.Extensions.AI; public static class RealtimeResponseStatus { /// - /// The response completed successfully. + /// Gets the status value indicating the response completed successfully. /// - public const string Completed = "completed"; + public static string Completed { get; } = "completed"; /// - /// The response was cancelled, typically due to an interruption such as user barge-in + /// Gets the status value indicating the response was cancelled, typically due to an interruption such as user barge-in /// (the user started speaking while the model was generating output). /// - public const string Cancelled = "cancelled"; + public static string Cancelled { get; } = "cancelled"; /// - /// The response ended before completing, for example because the output reached + /// Gets the status value indicating the response ended before completing, for example because the output reached /// the maximum token limit. /// - public const string Incomplete = "incomplete"; + public static string Incomplete { get; } = "incomplete"; /// - /// The response failed due to an error. + /// Gets the status value indicating the response failed due to an error. /// - public const string Failed = "failed"; + public static string Failed { get; } = "failed"; } From a18403b46a4dbb07e95f83fe5545570e6417680e Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 10 Mar 2026 17:50:05 -0700 Subject: [PATCH 90/92] Add GetRequiredService extension methods for realtime types Add GetRequiredService and GetRequiredService to both IRealtimeClient and IRealtimeClientSession extensions, matching the pattern from IChatClient and IEmbeddingGenerator. --- .../Realtime/RealtimeClientExtensions.cs | 51 +++++++ .../RealtimeClientSessionExtensions.cs | 51 +++++++ .../Microsoft.Extensions.AI/Throw.cs | 15 +++ .../Realtime/RealtimeClientExtensionsTests.cs | 124 ++++++++++++++++++ .../RealtimeClientSessionExtensionsTests.cs | 59 +++++++++ 5 files changed, 300 insertions(+) create mode 100644 src/Libraries/Microsoft.Extensions.AI/Throw.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientExtensionsTests.cs diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientExtensions.cs index 2efca1329f3..44837e26283 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientExtensions.cs @@ -28,4 +28,55 @@ public static class RealtimeClientExtensions return client.GetService(typeof(TService), serviceKey) is TService service ? service : default; } + + /// + /// Asks the for an object of the specified type + /// and throws an exception if one isn't available. + /// + /// The client. + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object. + /// is . + /// is . + /// No service of the requested type for the specified key is available. + /// + /// The purpose of this method is to allow for the retrieval of services that are required to be provided by the , + /// including itself or any services it might be wrapping. + /// + public static object GetRequiredService(this IRealtimeClient client, Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(client); + _ = Throw.IfNull(serviceType); + + return + client.GetService(serviceType, serviceKey) ?? + throw Throw.CreateMissingServiceException(serviceType, serviceKey); + } + + /// + /// Asks the for an object of type + /// and throws an exception if one isn't available. + /// + /// The type of the object to be retrieved. + /// The client. + /// An optional key that can be used to help identify the target service. + /// The found object. + /// is . + /// No service of the requested type for the specified key is available. + /// + /// The purpose of this method is to allow for the retrieval of strongly typed services that are required to be provided by the , + /// including itself or any services it might be wrapping. + /// + public static TService GetRequiredService(this IRealtimeClient client, object? serviceKey = null) + { + _ = Throw.IfNull(client); + + if (client.GetService(typeof(TService), serviceKey) is not TService service) + { + throw Throw.CreateMissingServiceException(typeof(TService), serviceKey); + } + + return service; + } } diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionExtensions.cs index 5e71973d4fb..4e4ccbd21ef 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/RealtimeClientSessionExtensions.cs @@ -28,4 +28,55 @@ public static class RealtimeClientSessionExtensions return session.GetService(typeof(TService), serviceKey) is TService service ? service : default; } + + /// + /// Asks the for an object of the specified type + /// and throws an exception if one isn't available. + /// + /// The session. + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object. + /// is . + /// is . + /// No service of the requested type for the specified key is available. + /// + /// The purpose of this method is to allow for the retrieval of services that are required to be provided by the , + /// including itself or any services it might be wrapping. + /// + public static object GetRequiredService(this IRealtimeClientSession session, Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(session); + _ = Throw.IfNull(serviceType); + + return + session.GetService(serviceType, serviceKey) ?? + throw Throw.CreateMissingServiceException(serviceType, serviceKey); + } + + /// + /// Asks the for an object of type + /// and throws an exception if one isn't available. + /// + /// The type of the object to be retrieved. + /// The session. + /// An optional key that can be used to help identify the target service. + /// The found object. + /// is . + /// No service of the requested type for the specified key is available. + /// + /// The purpose of this method is to allow for the retrieval of strongly typed services that are required to be provided by the , + /// including itself or any services it might be wrapping. + /// + public static TService GetRequiredService(this IRealtimeClientSession session, object? serviceKey = null) + { + _ = Throw.IfNull(session); + + if (session.GetService(typeof(TService), serviceKey) is not TService service) + { + throw Throw.CreateMissingServiceException(typeof(TService), serviceKey); + } + + return service; + } } diff --git a/src/Libraries/Microsoft.Extensions.AI/Throw.cs b/src/Libraries/Microsoft.Extensions.AI/Throw.cs new file mode 100644 index 00000000000..0d8f0db7fe5 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Throw.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Shared.Diagnostics; + +internal static partial class Throw +{ + /// Throws an exception indicating that a required service is not available. + public static InvalidOperationException CreateMissingServiceException(Type serviceType, object? serviceKey) => + new InvalidOperationException(serviceKey is null ? + $"No service of type '{serviceType}' is available." : + $"No service of type '{serviceType}' for the key '{serviceKey}' is available."); +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientExtensionsTests.cs new file mode 100644 index 00000000000..aed335164f0 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientExtensionsTests.cs @@ -0,0 +1,124 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +#pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. + +namespace Microsoft.Extensions.AI; + +public class RealtimeClientExtensionsTests +{ + [Fact] + public void GetService_NullClient_Throws() + { + Assert.Throws("client", () => ((IRealtimeClient)null!).GetService()); + } + + [Fact] + public void GetService_ReturnsMatchingService() + { + using var client = new TestRealtimeClient(); + var result = client.GetService(); + Assert.Same(client, result); + } + + [Fact] + public void GetService_ReturnsNullForNonMatchingType() + { + using var client = new TestRealtimeClient(); + var result = client.GetService(); + Assert.Null(result); + } + + [Fact] + public void GetService_WithServiceKey_ReturnsNull() + { + using var client = new TestRealtimeClient(); + var result = client.GetService("someKey"); + Assert.Null(result); + } + + [Fact] + public void GetService_ReturnsInterfaceType() + { + using var client = new TestRealtimeClient(); + var result = client.GetService(); + Assert.Same(client, result); + } + + [Fact] + public void GetRequiredService_NullClient_Throws() + { + Assert.Throws("client", () => ((IRealtimeClient)null!).GetRequiredService(typeof(string))); + Assert.Throws("client", () => ((IRealtimeClient)null!).GetRequiredService()); + } + + [Fact] + public void GetRequiredService_NullServiceType_Throws() + { + using var client = new TestRealtimeClient(); + Assert.Throws("serviceType", () => client.GetRequiredService(null!)); + } + + [Fact] + public void GetRequiredService_ReturnsMatchingService() + { + using var client = new TestRealtimeClient(); + var result = client.GetRequiredService(); + Assert.Same(client, result); + } + + [Fact] + public void GetRequiredService_ReturnsInterfaceType() + { + using var client = new TestRealtimeClient(); + var result = client.GetRequiredService(); + Assert.Same(client, result); + } + + [Fact] + public void GetRequiredService_NonGeneric_ReturnsMatchingService() + { + using var client = new TestRealtimeClient(); + var result = client.GetRequiredService(typeof(TestRealtimeClient)); + Assert.Same(client, result); + } + + [Fact] + public void GetRequiredService_ThrowsForNonMatchingType() + { + using var client = new TestRealtimeClient(); + Assert.Throws(() => client.GetRequiredService()); + } + + [Fact] + public void GetRequiredService_NonGeneric_ThrowsForNonMatchingType() + { + using var client = new TestRealtimeClient(); + Assert.Throws(() => client.GetRequiredService(typeof(string))); + } + + [Fact] + public void GetRequiredService_WithServiceKey_ThrowsForNonMatchingKey() + { + using var client = new TestRealtimeClient(); + Assert.Throws(() => client.GetRequiredService("someKey")); + } + + private sealed class TestRealtimeClient : IRealtimeClient + { + public Task CreateSessionAsync(RealtimeSessionOptions? options = null, CancellationToken cancellationToken = default) + => Task.FromResult(new TestRealtimeClientSession()); + + public object? GetService(Type serviceType, object? serviceKey = null) => + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : null; + + public void Dispose() + { + } + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientSessionExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientSessionExtensionsTests.cs index 3970e8e8010..e9ef04f8f8c 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientSessionExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Realtime/RealtimeClientSessionExtensionsTests.cs @@ -48,4 +48,63 @@ public async Task GetService_ReturnsInterfaceType() var result = session.GetService(); Assert.Same(session, result); } + + [Fact] + public void GetRequiredService_NullSession_Throws() + { + Assert.Throws("session", () => ((IRealtimeClientSession)null!).GetRequiredService(typeof(string))); + Assert.Throws("session", () => ((IRealtimeClientSession)null!).GetRequiredService()); + } + + [Fact] + public async Task GetRequiredService_NullServiceType_Throws() + { + await using var session = new TestRealtimeClientSession(); + Assert.Throws("serviceType", () => session.GetRequiredService(null!)); + } + + [Fact] + public async Task GetRequiredService_ReturnsMatchingService() + { + await using var session = new TestRealtimeClientSession(); + var result = session.GetRequiredService(); + Assert.Same(session, result); + } + + [Fact] + public async Task GetRequiredService_ReturnsInterfaceType() + { + await using var session = new TestRealtimeClientSession(); + var result = session.GetRequiredService(); + Assert.Same(session, result); + } + + [Fact] + public async Task GetRequiredService_NonGeneric_ReturnsMatchingService() + { + await using var session = new TestRealtimeClientSession(); + var result = session.GetRequiredService(typeof(TestRealtimeClientSession)); + Assert.Same(session, result); + } + + [Fact] + public async Task GetRequiredService_ThrowsForNonMatchingType() + { + await using var session = new TestRealtimeClientSession(); + Assert.Throws(() => session.GetRequiredService()); + } + + [Fact] + public async Task GetRequiredService_NonGeneric_ThrowsForNonMatchingType() + { + await using var session = new TestRealtimeClientSession(); + Assert.Throws(() => session.GetRequiredService(typeof(string))); + } + + [Fact] + public async Task GetRequiredService_WithServiceKey_ThrowsForNonMatchingKey() + { + await using var session = new TestRealtimeClientSession(); + Assert.Throws(() => session.GetRequiredService("someKey")); + } } From 76fdedd52692c6c4fbed4006b9e807c1f13c7a51 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 10 Mar 2026 17:56:28 -0700 Subject: [PATCH 91/92] Remove unused System.Threading.Channels package reference --- .../Microsoft.Extensions.AI.OpenAI.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj b/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj index 89ff9b90c29..dfd21f5451f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj @@ -37,7 +37,6 @@ - From 0dc6bcad1b9100fbebacebb41692c21539b602cb Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Tue, 10 Mar 2026 18:07:37 -0700 Subject: [PATCH 92/92] Update OTel semantic conventions version reference from v1.39 to v1.40 --- src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs | 2 +- .../Realtime/OpenTelemetryRealtimeClientSession.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs index 67a99febe8d..27d0ddab0d5 100644 --- a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs +++ b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs @@ -172,7 +172,7 @@ public static class Usage /// /// Custom attributes for realtime sessions. - /// These attributes are NOT part of the OpenTelemetry GenAI semantic conventions (as of v1.39). + /// These attributes are NOT part of the OpenTelemetry GenAI semantic conventions (as of v1.40). /// They are custom extensions to capture realtime session-specific configuration. /// public static class Realtime diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs index 09586c4d218..d8cd411f020 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs @@ -57,7 +57,7 @@ namespace Microsoft.Extensions.AI; /// /// /// -/// Additionally, the following custom attributes are supported (not part of OpenTelemetry GenAI semantic conventions as of v1.39): +/// Additionally, the following custom attributes are supported (not part of OpenTelemetry GenAI semantic conventions as of v1.40): /// /// gen_ai.request.tool_choice - Tool choice mode ("none", "auto", "required") or specific tool name /// gen_ai.realtime.voice - Voice setting from options