voice: add response_format to ModelSettings and chat()

Author: vishal-seshagiri-infinitusai

livekit-agents/livekit/agents/llm/fallback_adapter.py

@@ -92,6 +92,7 @@ def chat(
         conn_options: APIConnectOptions = DEFAULT_FALLBACK_API_CONNECT_OPTIONS,
         parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN,
         tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN,
+        response_format: NotGivenOr[Any] = NOT_GIVEN,
         extra_kwargs: NotGivenOr[dict[str, Any]] = NOT_GIVEN,
     ) -> LLMStream:
         return FallbackLLMStream(
@@ -101,6 +102,7 @@ def chat(
             tools=tools or [],
             parallel_tool_calls=parallel_tool_calls,
             tool_choice=tool_choice,
+            response_format=response_format,
             extra_kwargs=extra_kwargs,
         )
 
@@ -124,12 +126,14 @@ def __init__(
         conn_options: APIConnectOptions,
         parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN,
         tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN,
+        response_format: NotGivenOr[Any] = NOT_GIVEN,
         extra_kwargs: NotGivenOr[dict[str, Any]] = NOT_GIVEN,
     ) -> None:
         super().__init__(llm, chat_ctx=chat_ctx, tools=tools, conn_options=conn_options)
         self._fallback_adapter = llm
         self._parallel_tool_calls = parallel_tool_calls
         self._tool_choice = tool_choice
+        self._response_format = response_format
         self._extra_kwargs = extra_kwargs
 
         self._current_stream: LLMStream | None = None
@@ -164,6 +168,7 @@ async def _try_generate(
                 tools=self._tools,
                 parallel_tool_calls=self._parallel_tool_calls,
                 tool_choice=self._tool_choice,
+                response_format=self._response_format,
                 extra_kwargs=self._extra_kwargs,
                 conn_options=dataclasses.replace(
                     self._conn_options,

livekit-agents/livekit/agents/llm/llm.py

@@ -128,6 +128,7 @@ def chat(
         conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
         parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN,
         tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN,
+        response_format: NotGivenOr[Any] = NOT_GIVEN,
         extra_kwargs: NotGivenOr[dict[str, Any]] = NOT_GIVEN,
     ) -> LLMStream: ...
 

livekit-agents/livekit/agents/voice/agent.py

@@ -29,6 +29,8 @@
 class ModelSettings:
     tool_choice: NotGivenOr[llm.ToolChoice] = NOT_GIVEN
     """The tool choice to use when calling the LLM."""
+    response_format: NotGivenOr[Any] = NOT_GIVEN
+    """The response format to use when calling the LLM."""
 
 
 class Agent:
@@ -409,11 +411,16 @@ async def llm_node(
             )
 
             tool_choice = model_settings.tool_choice if model_settings else NOT_GIVEN
+            response_format = model_settings.response_format if model_settings else NOT_GIVEN
             activity_llm = activity.llm
 
             conn_options = activity.session.conn_options.llm_conn_options
             async with activity_llm.chat(
-                chat_ctx=chat_ctx, tools=tools, tool_choice=tool_choice, conn_options=conn_options
+                chat_ctx=chat_ctx,
+                tools=tools,
+                tool_choice=tool_choice,
+                response_format=response_format,
+                conn_options=conn_options,
             ) as stream:
                 async for chunk in stream:
                     yield chunk

livekit-plugins/livekit-plugins-anthropic/livekit/plugins/anthropic/llm.py

@@ -130,6 +130,7 @@ def chat(
         conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
         parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN,
         tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN,
+        response_format: NotGivenOr[Any] = NOT_GIVEN,
         extra_kwargs: NotGivenOr[dict[str, Any]] = NOT_GIVEN,
     ) -> LLMStream:
         extra = {}

livekit-plugins/livekit-plugins-aws/livekit/plugins/aws/llm.py

@@ -131,6 +131,7 @@ def chat(
         conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
         tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN,
         temperature: NotGivenOr[float] = NOT_GIVEN,
+        response_format: NotGivenOr[Any] = NOT_GIVEN,
         extra_kwargs: NotGivenOr[dict[str, Any]] = NOT_GIVEN,
     ) -> LLMStream:
         opts: dict[str, Any] = {}

livekit-plugins/livekit-plugins-langchain/livekit/plugins/langchain/langgraph.py

@@ -64,6 +64,7 @@ def chat(
         # these are unused, since tool execution takes place in langgraph
         parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN,
         tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN,
+        response_format: NotGivenOr[Any] = NOT_GIVEN,
         extra_kwargs: NotGivenOr[dict[str, Any]] = NOT_GIVEN,
     ) -> LangGraphStream[ContextT]:
         return LangGraphStream(

livekit-plugins/livekit-plugins-mistralai/livekit/plugins/mistralai/llm.py

@@ -90,9 +90,6 @@ def chat(
         if is_given(tool_choice):
             extra["tool_choice"] = tool_choice
 
-        if is_given(response_format):
-            extra["response_format"] = response_format
-
         return LLMStream(
             self,
             model=self._opts.model,

livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/responses/llm.py

@@ -119,6 +119,7 @@ def chat(
         conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
         parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN,
         tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN,
+        response_format: NotGivenOr[Any] = NOT_GIVEN,
         extra_kwargs: NotGivenOr[dict[str, Any]] = NOT_GIVEN,
     ) -> LLMStream:
         extra = {}

tests/fake_llm.py

@@ -62,6 +62,7 @@ def chat(
         conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
         parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN,
         tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN,
+        response_format: NotGivenOr[Any] = NOT_GIVEN,
         extra_kwargs: NotGivenOr[dict[str, Any]] = NOT_GIVEN,
     ) -> LLMStream:
         return FakeLLMStream(self, chat_ctx=chat_ctx, tools=tools or [], conn_options=conn_options)