Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
545 changes: 545 additions & 0 deletions src/__tests__/content-with-toolcalls.test.ts

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions src/__tests__/stream-collapse.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1568,7 +1568,7 @@ describe("collapseOllamaNDJSON with tool_calls", () => {
expect(result.content).toBeUndefined();
});

it("returns toolCalls (not content) when both tool_calls and text are present", () => {
it("preserves both content and toolCalls when both tool_calls and text are present", () => {
const body = [
JSON.stringify({
model: "llama3",
Expand All @@ -1594,11 +1594,11 @@ describe("collapseOllamaNDJSON with tool_calls", () => {
].join("\n");

const result = collapseOllamaNDJSON(body);
// When toolCalls are present, they take priority over content
// When toolCalls are present alongside content, both are preserved
expect(result.toolCalls).toBeDefined();
expect(result.toolCalls).toHaveLength(1);
expect(result.toolCalls![0].name).toBe("get_weather");
expect(result.content).toBeUndefined();
expect(result.content).toBe("Let me check the weather.");
});

it("extracts multiple tool_calls across chunks", () => {
Expand Down
156 changes: 130 additions & 26 deletions src/gemini.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import type {
import {
isTextResponse,
isToolCallResponse,
isContentWithToolCallsResponse,
isErrorResponse,
generateToolCallId,
flattenHeaders,
Expand Down Expand Up @@ -256,24 +257,22 @@ function buildGeminiTextStreamChunks(
return chunks;
}

function parseToolCallPart(tc: ToolCall, logger: Logger): GeminiPart {
let argsObj: Record<string, unknown>;
try {
argsObj = JSON.parse(tc.arguments || "{}") as Record<string, unknown>;
} catch {
logger.warn(`Malformed JSON in fixture tool call arguments for "${tc.name}": ${tc.arguments}`);
argsObj = {};
}
return { functionCall: { name: tc.name, args: argsObj, id: tc.id || generateToolCallId() } };
}

function buildGeminiToolCallStreamChunks(
toolCalls: ToolCall[],
logger: Logger,
): GeminiResponseChunk[] {
const parts: GeminiPart[] = toolCalls.map((tc) => {
let argsObj: Record<string, unknown>;
try {
argsObj = JSON.parse(tc.arguments || "{}") as Record<string, unknown>;
} catch {
logger.warn(
`Malformed JSON in fixture tool call arguments for "${tc.name}": ${tc.arguments}`,
);
argsObj = {};
}
return {
functionCall: { name: tc.name, args: argsObj, id: tc.id || generateToolCallId() },
};
});
const parts: GeminiPart[] = toolCalls.map((tc) => parseToolCallPart(tc, logger));

// Gemini sends all tool calls in a single response chunk
return [
Expand Down Expand Up @@ -320,21 +319,85 @@ function buildGeminiTextResponse(content: string, reasoning?: string): GeminiRes
}

function buildGeminiToolCallResponse(toolCalls: ToolCall[], logger: Logger): GeminiResponseChunk {
const parts: GeminiPart[] = toolCalls.map((tc) => {
let argsObj: Record<string, unknown>;
try {
argsObj = JSON.parse(tc.arguments || "{}") as Record<string, unknown>;
} catch {
logger.warn(
`Malformed JSON in fixture tool call arguments for "${tc.name}": ${tc.arguments}`,
);
argsObj = {};
const parts: GeminiPart[] = toolCalls.map((tc) => parseToolCallPart(tc, logger));

return {
candidates: [
{
content: { role: "model", parts },
finishReason: "FUNCTION_CALL",
index: 0,
},
],
usageMetadata: {
promptTokenCount: 0,
candidatesTokenCount: 0,
totalTokenCount: 0,
},
};
}

function buildGeminiContentWithToolCallsStreamChunks(
content: string,
toolCalls: ToolCall[],
chunkSize: number,
logger: Logger,
): GeminiResponseChunk[] {
const chunks: GeminiResponseChunk[] = [];

if (content.length === 0) {
chunks.push({
candidates: [
{
content: { role: "model", parts: [{ text: "" }] },
index: 0,
},
],
});
} else {
for (let i = 0; i < content.length; i += chunkSize) {
const slice = content.slice(i, i + chunkSize);
chunks.push({
candidates: [
{
content: { role: "model", parts: [{ text: slice }] },
index: 0,
},
],
});
}
return {
functionCall: { name: tc.name, args: argsObj, id: tc.id || generateToolCallId() },
};
}

const parts: GeminiPart[] = toolCalls.map((tc) => parseToolCallPart(tc, logger));

chunks.push({
candidates: [
{
content: { role: "model", parts },
finishReason: "FUNCTION_CALL",
index: 0,
},
],
usageMetadata: {
promptTokenCount: 0,
candidatesTokenCount: 0,
totalTokenCount: 0,
},
});

return chunks;
}

function buildGeminiContentWithToolCallsResponse(
content: string,
toolCalls: ToolCall[],
logger: Logger,
): GeminiResponseChunk {
const parts: GeminiPart[] = [
{ text: content },
...toolCalls.map((tc) => parseToolCallPart(tc, logger)),
];

return {
candidates: [
{
Expand Down Expand Up @@ -549,6 +612,47 @@ export async function handleGemini(
return;
}

// Content + tool calls response (must be checked before isTextResponse / isToolCallResponse)
if (isContentWithToolCallsResponse(response)) {
const journalEntry = journal.add({
method: req.method ?? "POST",
path,
headers: flattenHeaders(req.headers),
body: completionReq,
response: { status: 200, fixture },
});
if (!streaming) {
const body = buildGeminiContentWithToolCallsResponse(
response.content,
response.toolCalls,
logger,
);
res.writeHead(200, { "Content-Type": "application/json" });
res.end(JSON.stringify(body));
} else {
const chunks = buildGeminiContentWithToolCallsStreamChunks(
response.content,
response.toolCalls,
chunkSize,
logger,
);
const interruption = createInterruptionSignal(fixture);
const completed = await writeGeminiSSEStream(res, chunks, {
latency,
streamingProfile: fixture.streamingProfile,
signal: interruption?.signal,
onChunkSent: interruption?.tick,
});
if (!completed) {
if (!res.writableEnded) res.destroy();
journalEntry.response.interrupted = true;
journalEntry.response.interruptReason = interruption?.reason();
}
interruption?.cleanup();
}
return;
}

// Text response
if (isTextResponse(response)) {
const journalEntry = journal.add({
Expand Down
136 changes: 136 additions & 0 deletions src/helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import type {
FixtureResponse,
TextResponse,
ToolCallResponse,
ContentWithToolCallsResponse,
ErrorResponse,
EmbeddingResponse,
SSEChunk,
Expand Down Expand Up @@ -50,6 +51,17 @@ export function isToolCallResponse(r: FixtureResponse): r is ToolCallResponse {
return "toolCalls" in r && Array.isArray((r as ToolCallResponse).toolCalls);
}

export function isContentWithToolCallsResponse(
r: FixtureResponse,
): r is ContentWithToolCallsResponse {
return (
"content" in r &&
typeof (r as ContentWithToolCallsResponse).content === "string" &&
"toolCalls" in r &&
Array.isArray((r as ContentWithToolCallsResponse).toolCalls)
);
}

export function isErrorResponse(r: FixtureResponse): r is ErrorResponse {
return (
"error" in r &&
Expand Down Expand Up @@ -254,6 +266,130 @@ export function buildToolCallCompletion(toolCalls: ToolCall[], model: string): C
};
}

export function buildContentWithToolCallsChunks(
content: string,
toolCalls: ToolCall[],
model: string,
chunkSize: number,
): SSEChunk[] {
const id = generateId();
const created = Math.floor(Date.now() / 1000);
const chunks: SSEChunk[] = [];

// Role chunk
chunks.push({
id,
object: "chat.completion.chunk",
created,
model,
choices: [{ index: 0, delta: { role: "assistant", content: "" }, finish_reason: null }],
});

// Content chunks
for (let i = 0; i < content.length; i += chunkSize) {
const slice = content.slice(i, i + chunkSize);
chunks.push({
id,
object: "chat.completion.chunk",
created,
model,
choices: [{ index: 0, delta: { content: slice }, finish_reason: null }],
});
}

// Tool call chunks — one initial chunk per tool call, then argument chunks
for (let tcIdx = 0; tcIdx < toolCalls.length; tcIdx++) {
const tc = toolCalls[tcIdx];
const tcId = tc.id || generateToolCallId();

// Initial tool call chunk (id + function name)
chunks.push({
id,
object: "chat.completion.chunk",
created,
model,
choices: [
{
index: 0,
delta: {
tool_calls: [
{
index: tcIdx,
id: tcId,
type: "function",
function: { name: tc.name, arguments: "" },
},
],
},
finish_reason: null,
},
],
});

// Argument streaming chunks
const args = tc.arguments;
for (let i = 0; i < args.length; i += chunkSize) {
const slice = args.slice(i, i + chunkSize);
chunks.push({
id,
object: "chat.completion.chunk",
created,
model,
choices: [
{
index: 0,
delta: {
tool_calls: [{ index: tcIdx, function: { arguments: slice } }],
},
finish_reason: null,
},
],
});
}
}

// Finish chunk
chunks.push({
id,
object: "chat.completion.chunk",
created,
model,
choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }],
});

return chunks;
}

export function buildContentWithToolCallsCompletion(
content: string,
toolCalls: ToolCall[],
model: string,
): ChatCompletion {
return {
id: generateId(),
object: "chat.completion",
created: Math.floor(Date.now() / 1000),
model,
choices: [
{
index: 0,
message: {
role: "assistant",
content,
refusal: null,
tool_calls: toolCalls.map((tc) => ({
id: tc.id || generateToolCallId(),
type: "function" as const,
function: { name: tc.name, arguments: tc.arguments },
})),
},
finish_reason: "tool_calls",
},
],
usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
};
}

// ─── HTTP helpers ─────────────────────────────────────────────────────────

export function readBody(req: http.IncomingMessage): Promise<string> {
Expand Down
Loading
Loading