From 86167a1d304b806ad999efc5a453a6300e1dcf8a Mon Sep 17 00:00:00 2001 From: lihq1403 Date: Mon, 26 May 2025 11:08:57 +0800 Subject: [PATCH 01/10] feat: Add streamIncludeUsage property to models and update ChatCompletionRequest for usage tracking --- src/Api/Request/ChatCompletionRequest.php | 12 ++++++++++++ src/Api/Response/ChatCompletionStreamResponse.php | 3 +++ src/Model/AbstractModel.php | 3 +++ src/Model/AzureOpenAIModel.php | 2 ++ src/Model/DoubaoModel.php | 2 ++ src/Model/OpenAIModel.php | 2 ++ src/Model/QianFanModel.php | 2 ++ 7 files changed, 26 insertions(+) diff --git a/src/Api/Request/ChatCompletionRequest.php b/src/Api/Request/ChatCompletionRequest.php index cba782c..22fa42c 100644 --- a/src/Api/Request/ChatCompletionRequest.php +++ b/src/Api/Request/ChatCompletionRequest.php @@ -47,6 +47,8 @@ class ChatCompletionRequest implements RequestInterface */ private ?int $totalTokenEstimate = null; + private bool $streamIncludeUsage = false; + public function __construct( /** @var MessageInterface[] $messages */ protected array $messages, @@ -101,6 +103,11 @@ public function createOptions(): array if (! empty($this->businessParams)) { $json['business_params'] = $this->businessParams; } + if ($this->stream && $this->streamIncludeUsage) { + $json['stream_options'] = [ + 'include_usage' => true, + ]; + } return [ RequestOptions::JSON => $json, @@ -181,6 +188,11 @@ public function setStreamContentEnabled(bool $streamContentEnabled): void $this->streamContentEnabled = $streamContentEnabled; } + public function setStreamIncludeUsage(bool $streamIncludeUsage): void + { + $this->streamIncludeUsage = $streamIncludeUsage; + } + /** * 获取消息列表. * diff --git a/src/Api/Response/ChatCompletionStreamResponse.php b/src/Api/Response/ChatCompletionStreamResponse.php index 19f4771..dc27c26 100644 --- a/src/Api/Response/ChatCompletionStreamResponse.php +++ b/src/Api/Response/ChatCompletionStreamResponse.php @@ -252,6 +252,9 @@ private function updateMetadata(array $data): void $this->setCreated($data['created'] ?? null); $this->setModel($data['model'] ?? null); $this->setChoices($data['choices'] ?? []); + if (! empty($data['usage'])) { + $this->setUsage(Usage::fromArray($data['usage'])); + } } /** diff --git a/src/Model/AbstractModel.php b/src/Model/AbstractModel.php index ea9a92b..6d72214 100644 --- a/src/Model/AbstractModel.php +++ b/src/Model/AbstractModel.php @@ -56,6 +56,8 @@ abstract class AbstractModel implements ModelInterface, EmbeddingInterface */ protected ModelOptions $modelOptions; + protected bool $streamIncludeUsage = false; + /** * 构造函数. */ @@ -132,6 +134,7 @@ public function chatStream( $chatRequest->setFrequencyPenalty($frequencyPenalty); $chatRequest->setPresencePenalty($presencePenalty); $chatRequest->setBusinessParams($businessParams); + $chatRequest->setStreamIncludeUsage($this->streamIncludeUsage); return $client->chatCompletionsStream($chatRequest); } catch (Throwable $e) { $context = $this->createErrorContext([ diff --git a/src/Model/AzureOpenAIModel.php b/src/Model/AzureOpenAIModel.php index 8734274..a926ba9 100644 --- a/src/Model/AzureOpenAIModel.php +++ b/src/Model/AzureOpenAIModel.php @@ -20,6 +20,8 @@ */ class AzureOpenAIModel extends AbstractModel { + protected bool $streamIncludeUsage = true; + /** * 获取Azure OpenAI客户端实例. */ diff --git a/src/Model/DoubaoModel.php b/src/Model/DoubaoModel.php index 32ac829..ec4224e 100644 --- a/src/Model/DoubaoModel.php +++ b/src/Model/DoubaoModel.php @@ -21,6 +21,8 @@ */ class DoubaoModel extends AbstractModel { + protected bool $streamIncludeUsage = true; + /** * 获取Doubao客户端实例. */ diff --git a/src/Model/OpenAIModel.php b/src/Model/OpenAIModel.php index 39b6711..b1b41f2 100644 --- a/src/Model/OpenAIModel.php +++ b/src/Model/OpenAIModel.php @@ -20,6 +20,8 @@ */ class OpenAIModel extends AbstractModel { + protected bool $streamIncludeUsage = true; + /** * 获取OpenAI客户端实例. */ diff --git a/src/Model/QianFanModel.php b/src/Model/QianFanModel.php index 4c8124a..751c35c 100644 --- a/src/Model/QianFanModel.php +++ b/src/Model/QianFanModel.php @@ -20,6 +20,8 @@ class QianFanModel extends AbstractModel { + protected bool $streamIncludeUsage = true; + public function embeddings(array|string $input, ?string $encoding_format = 'float', ?string $user = null): EmbeddingResponse { try { From 0224f62fc9c762997cbd48e13aca6c655fdb364f Mon Sep 17 00:00:00 2001 From: lihq1403 Date: Mon, 26 May 2025 14:24:53 +0800 Subject: [PATCH 02/10] feat: Enhance chat completion handling with business parameters and event dispatching --- src/Api/Providers/AbstractClient.php | 6 + src/Api/Request/ChatCompletionRequest.php | 14 +- src/Api/Request/CompletionRequest.php | 9 +- .../Response/ChatCompletionStreamResponse.php | 164 +++++++++++++++++- src/Event/AfterChatCompletionsEvent.php | 25 +++ src/Event/AfterChatCompletionsStreamEvent.php | 65 +++++++ src/Model/AbstractModel.php | 5 + src/Utils/EventUtil.php | 30 ++++ 8 files changed, 314 insertions(+), 4 deletions(-) create mode 100644 src/Event/AfterChatCompletionsEvent.php create mode 100644 src/Event/AfterChatCompletionsStreamEvent.php create mode 100644 src/Utils/EventUtil.php diff --git a/src/Api/Providers/AbstractClient.php b/src/Api/Providers/AbstractClient.php index 3a14a17..7e8ce10 100644 --- a/src/Api/Providers/AbstractClient.php +++ b/src/Api/Providers/AbstractClient.php @@ -25,10 +25,13 @@ use Hyperf\Odin\Api\Transport\SSEClient; use Hyperf\Odin\Contract\Api\ClientInterface; use Hyperf\Odin\Contract\Api\ConfigInterface; +use Hyperf\Odin\Event\AfterChatCompletionsEvent; +use Hyperf\Odin\Event\AfterChatCompletionsStreamEvent; use Hyperf\Odin\Exception\LLMException; use Hyperf\Odin\Exception\LLMException\ErrorHandlerInterface; use Hyperf\Odin\Exception\LLMException\ErrorMappingManager; use Hyperf\Odin\Exception\LLMException\LLMErrorHandler; +use Hyperf\Odin\Utils\EventUtil; use Psr\Log\LoggerInterface; use Throwable; @@ -86,6 +89,8 @@ public function chatCompletions(ChatCompletionRequest $chatRequest): ChatComplet 'content' => $chatCompletionResponse->getContent(), ]); + EventUtil::dispatch(new AfterChatCompletionsEvent($chatRequest, $chatCompletionResponse, $duration)); + return $chatCompletionResponse; } catch (Throwable $e) { throw $this->convertException($e, [ @@ -125,6 +130,7 @@ public function chatCompletionsStream(ChatCompletionRequest $chatRequest): ChatC ); $chatCompletionStreamResponse = new ChatCompletionStreamResponse($response, $this->logger, $sseClient); + $chatCompletionStreamResponse->setAfterChatCompletionsStreamEvent(new AfterChatCompletionsStreamEvent($chatRequest, $firstResponseDuration)); $this->logger?->debug('ChatCompletionsStreamResponse', [ 'first_response_ms' => $firstResponseDuration, diff --git a/src/Api/Request/ChatCompletionRequest.php b/src/Api/Request/ChatCompletionRequest.php index 22fa42c..952b7e2 100644 --- a/src/Api/Request/ChatCompletionRequest.php +++ b/src/Api/Request/ChatCompletionRequest.php @@ -31,6 +31,8 @@ class ChatCompletionRequest implements RequestInterface private float $presencePenalty = 0.0; + private bool $includeBusinessParams = false; + private array $businessParams = []; private bool $toolsCache = false; @@ -100,7 +102,7 @@ public function createOptions(): array if ($this->presencePenalty > 0) { $json['presence_penalty'] = $this->presencePenalty; } - if (! empty($this->businessParams)) { + if ($this->includeBusinessParams && ! empty($this->businessParams)) { $json['business_params'] = $this->businessParams; } if ($this->stream && $this->streamIncludeUsage) { @@ -168,6 +170,16 @@ public function setBusinessParams(array $businessParams): void $this->businessParams = $businessParams; } + public function getBusinessParams(): array + { + return $this->businessParams; + } + + public function setIncludeBusinessParams(bool $includeBusinessParams): void + { + $this->includeBusinessParams = $includeBusinessParams; + } + public function setStream(bool $stream): void { $this->stream = $stream; diff --git a/src/Api/Request/CompletionRequest.php b/src/Api/Request/CompletionRequest.php index bfabbbd..ad5061c 100644 --- a/src/Api/Request/CompletionRequest.php +++ b/src/Api/Request/CompletionRequest.php @@ -24,6 +24,8 @@ class CompletionRequest implements RequestInterface private array $businessParams = []; + private bool $includeBusinessParams = false; + public function __construct( protected string $model, protected string $prompt, @@ -65,7 +67,7 @@ public function createOptions(): array if ($this->presencePenalty > 0) { $json['presence_penalty'] = $this->presencePenalty; } - if (! empty($this->businessParams)) { + if ($this->includeBusinessParams && ! empty($this->businessParams)) { $json['business_params'] = $this->businessParams; } @@ -89,6 +91,11 @@ public function setBusinessParams(array $businessParams): void $this->businessParams = $businessParams; } + public function setIncludeBusinessParams(bool $includeBusinessParams): void + { + $this->includeBusinessParams = $includeBusinessParams; + } + /** * 获取模型名称. * diff --git a/src/Api/Response/ChatCompletionStreamResponse.php b/src/Api/Response/ChatCompletionStreamResponse.php index dc27c26..fe43bf5 100644 --- a/src/Api/Response/ChatCompletionStreamResponse.php +++ b/src/Api/Response/ChatCompletionStreamResponse.php @@ -16,7 +16,10 @@ use GuzzleHttp\Psr7\Response; use Hyperf\Odin\Api\Transport\SSEClient; use Hyperf\Odin\Api\Transport\SSEEvent; +use Hyperf\Odin\Event\AfterChatCompletionsStreamEvent; use Hyperf\Odin\Exception\LLMException; +use Hyperf\Odin\Message\AssistantMessage; +use Hyperf\Odin\Utils\EventUtil; use IteratorAggregate; use JsonException; use Psr\Http\Message\ResponseInterface as PsrResponseInterface; @@ -34,6 +37,9 @@ class ChatCompletionStreamResponse extends AbstractResponse implements Stringabl protected ?string $model = null; + /** + * @var array + */ protected array $choices = []; /** @@ -46,6 +52,8 @@ class ChatCompletionStreamResponse extends AbstractResponse implements Stringabl */ protected ?IteratorAggregate $iterator = null; + protected AfterChatCompletionsStreamEvent $afterChatCompletionsStreamEvent; + /** * 构造函数. * @@ -93,6 +101,11 @@ public function getStreamIterator(): Generator return $this->iterateWithLegacyMethod(); } + public function setAfterChatCompletionsStreamEvent(AfterChatCompletionsStreamEvent $afterChatCompletionsStreamEvent): void + { + $this->afterChatCompletionsStreamEvent = $afterChatCompletionsStreamEvent; + } + public function getId(): ?string { return $this->id; @@ -159,6 +172,7 @@ protected function parseContent(): self private function iterateWithCustomIterator(): Generator { try { + $startTime = microtime(true); foreach ($this->iterator->getIterator() as $data) { // 处理结束标记 if ($data === '[DONE]' || $data === json_encode('[DONE]')) { @@ -188,6 +202,9 @@ private function iterateWithCustomIterator(): Generator // 生成ChatCompletionChoice对象 yield from $this->yieldChoices($data['choices'] ?? []); } + + // Set duration and create completion response + $this->handleStreamCompletion($startTime); } catch (Throwable $e) { $this->logger?->error('Error processing custom iterator', [ 'exception' => get_class($e), @@ -204,6 +221,7 @@ private function iterateWithCustomIterator(): Generator private function iterateWithSSEClient(): Generator { try { + $startTime = microtime(true); /** @var SSEEvent $event */ foreach ($this->sseClient->getIterator() as $event) { $data = $event->getData(); @@ -232,6 +250,9 @@ private function iterateWithSSEClient(): Generator // 生成ChatCompletionChoice对象 yield from $this->yieldChoices($data['choices'] ?? []); } + + // Set duration and create completion response + $this->handleStreamCompletion($startTime); } catch (Throwable $e) { $this->logger?->error('Error processing SSE stream', [ 'exception' => get_class($e), @@ -251,7 +272,6 @@ private function updateMetadata(array $data): void $this->setObject($data['object'] ?? null); $this->setCreated($data['created'] ?? null); $this->setModel($data['model'] ?? null); - $this->setChoices($data['choices'] ?? []); if (! empty($data['usage'])) { $this->setUsage(Usage::fromArray($data['usage'])); } @@ -267,7 +287,9 @@ private function yieldChoices(array $choices): Generator $this->logger?->warning('Invalid choice format', ['choice' => $choice]); continue; } - yield ChatCompletionChoice::fromArray($choice); + $chatCompletionChoice = ChatCompletionChoice::fromArray($choice); + $this->choices[] = $chatCompletionChoice; + yield $chatCompletionChoice; } } @@ -277,6 +299,7 @@ private function yieldChoices(array $choices): Generator private function iterateWithLegacyMethod(): Generator { // 保留原有的实现作为后备 + $startTime = microtime(true); $body = $this->originResponse->getBody(); $buffer = ''; @@ -314,5 +337,142 @@ private function iterateWithLegacyMethod(): Generator } } } + + // Set duration and create completion response + $this->handleStreamCompletion($startTime); + } + + /** + * Handle stream completion - create response and dispatch event. + */ + private function handleStreamCompletion(float $startTime): void + { + // Set duration and create completion response + $this->afterChatCompletionsStreamEvent->setDuration(microtime(true) - $startTime); + + // Create and set the completed ChatCompletionResponse + $completionResponse = $this->createChatCompletionResponse(); + $this->afterChatCompletionsStreamEvent->setCompletionResponse($completionResponse); + + EventUtil::dispatch($this->afterChatCompletionsStreamEvent); + } + + private function createChatCompletionResponse(): ChatCompletionResponse + { + // Create a merged choices array by combining content from the same index + $mergedChoices = []; + + foreach ($this->choices as $choice) { + $index = $choice->getIndex() ?? 0; + + if (! isset($mergedChoices[$index])) { + // Initialize new choice with basic info + $mergedChoices[$index] = [ + 'index' => $index, + 'message' => [ + 'role' => 'assistant', + 'content' => '', + 'reasoning_content' => null, + 'tool_calls' => [], + ], + 'logprobs' => $choice->getLogprobs(), + 'finish_reason' => null, + ]; + } + + // Merge content + $message = $choice->getMessage(); + // Append content + $content = $message->getContent(); + if (! empty($content)) { + $mergedChoices[$index]['message']['content'] .= $content; + } + + // Handle reasoning content for AssistantMessage + if ($message instanceof AssistantMessage) { + $reasoningContent = $message->getReasoningContent(); + if (! empty($reasoningContent)) { + if ($mergedChoices[$index]['message']['reasoning_content'] === null) { + $mergedChoices[$index]['message']['reasoning_content'] = ''; + } + $mergedChoices[$index]['message']['reasoning_content'] .= $reasoningContent; + } + + // Merge tool calls + $toolCalls = $message->getToolCalls(); + if (! empty($toolCalls)) { + foreach ($toolCalls as $toolCall) { + $toolCallId = $toolCall->getId(); + $existingToolCallFound = false; + + // Check if this tool call already exists and merge stream arguments + foreach ($mergedChoices[$index]['message']['tool_calls'] as &$existingToolCall) { + if ($existingToolCall['id'] === $toolCallId) { + // Append stream arguments for existing tool call + if (isset($existingToolCall['function']['arguments'])) { + $existingToolCall['function']['arguments'] .= $toolCall->getStreamArguments(); + } else { + $existingToolCall['function']['arguments'] = $toolCall->getStreamArguments(); + } + $existingToolCallFound = true; + break; + } + } + + // Add new tool call if not found + if (! $existingToolCallFound) { + $mergedChoices[$index]['message']['tool_calls'][] = [ + 'id' => $toolCall->getId(), + 'type' => $toolCall->getType(), + 'function' => [ + 'name' => $toolCall->getName(), + 'arguments' => $toolCall->getStreamArguments() ?: json_encode($toolCall->getArguments()), + ], + ]; + } + } + } + } + + // Update finish reason if provided + if ($choice->getFinishReason()) { + $mergedChoices[$index]['finish_reason'] = $choice->getFinishReason(); + } + } + + // Clean up empty reasoning_content + foreach ($mergedChoices as &$choice) { + if (empty($choice['message']['reasoning_content'])) { + $choice['message']['reasoning_content'] = null; + } + if (empty($choice['message']['tool_calls'])) { + unset($choice['message']['tool_calls']); + } + } + + // Sort choices by index + ksort($mergedChoices); + $mergedChoices = array_values($mergedChoices); + + // Create response content similar to regular chat completion response + $responseContent = [ + 'id' => $this->getId(), + 'object' => $this->getObject() ?: 'chat.completion', + 'created' => $this->getCreated(), + 'model' => $this->getModel(), + 'choices' => $mergedChoices, + ]; + + // Add usage if available + if ($this->getUsage()) { + $responseContent['usage'] = $this->getUsage()->toArray(); + } + + // Create a mock response with the merged content + $jsonContent = json_encode($responseContent); + $mockResponse = new Response(200, ['Content-Type' => 'application/json'], $jsonContent); + + // Create and return ChatCompletionResponse + return new ChatCompletionResponse($mockResponse, $this->logger); } } diff --git a/src/Event/AfterChatCompletionsEvent.php b/src/Event/AfterChatCompletionsEvent.php new file mode 100644 index 0000000..23b339a --- /dev/null +++ b/src/Event/AfterChatCompletionsEvent.php @@ -0,0 +1,25 @@ +completionRequest = $completionRequest; + $this->firstResponseDuration = $firstResponseDuration; + } + + public function getCompletionRequest(): ChatCompletionRequest + { + return $this->completionRequest; + } + + public function setCompletionRequest(ChatCompletionRequest $completionRequest): void + { + $this->completionRequest = $completionRequest; + } + + public function getCompletionResponse(): ChatCompletionResponse + { + return $this->completionResponse; + } + + public function setCompletionResponse(ChatCompletionResponse $completionResponse): void + { + $this->completionResponse = $completionResponse; + } + + public function getDuration(): float + { + return $this->duration; + } + + public function setDuration(float $duration): void + { + $this->duration = $duration; + } +} diff --git a/src/Model/AbstractModel.php b/src/Model/AbstractModel.php index 6d72214..33a17bf 100644 --- a/src/Model/AbstractModel.php +++ b/src/Model/AbstractModel.php @@ -58,6 +58,8 @@ abstract class AbstractModel implements ModelInterface, EmbeddingInterface protected bool $streamIncludeUsage = false; + protected bool $includeBusinessParams = false; + /** * 构造函数. */ @@ -94,6 +96,7 @@ public function chat( $chatRequest->setFrequencyPenalty($frequencyPenalty); $chatRequest->setPresencePenalty($presencePenalty); $chatRequest->setBusinessParams($businessParams); + $chatRequest->setIncludeBusinessParams($this->includeBusinessParams); return $client->chatCompletions($chatRequest); } catch (Throwable $e) { $context = $this->createErrorContext([ @@ -135,6 +138,7 @@ public function chatStream( $chatRequest->setPresencePenalty($presencePenalty); $chatRequest->setBusinessParams($businessParams); $chatRequest->setStreamIncludeUsage($this->streamIncludeUsage); + $chatRequest->setIncludeBusinessParams($this->includeBusinessParams); return $client->chatCompletionsStream($chatRequest); } catch (Throwable $e) { $context = $this->createErrorContext([ @@ -169,6 +173,7 @@ public function completions( $chatRequest->setFrequencyPenalty($frequencyPenalty); $chatRequest->setPresencePenalty($presencePenalty); $chatRequest->setBusinessParams($businessParams); + $chatRequest->setIncludeBusinessParams($this->includeBusinessParams); return $client->completions($chatRequest); } diff --git a/src/Utils/EventUtil.php b/src/Utils/EventUtil.php new file mode 100644 index 0000000..7fd213c --- /dev/null +++ b/src/Utils/EventUtil.php @@ -0,0 +1,30 @@ +has(EventDispatcherInterface::class)) { + return; + } + $dispatcher = ApplicationContext::getContainer()->get(EventDispatcherInterface::class); + $dispatcher->dispatch($event); + } +} From 8bdf2e68ce18555df4c8540f40e351a10d0060b3 Mon Sep 17 00:00:00 2001 From: lihq1403 Date: Mon, 26 May 2025 14:39:25 +0800 Subject: [PATCH 03/10] feat: Add usage event formatting and enhance stream completion handling --- .../AwsBedrockConverseFormatConverter.php | 29 +++++++++++++++++++ .../Response/ChatCompletionStreamResponse.php | 4 +++ 2 files changed, 33 insertions(+) diff --git a/src/Api/Providers/AwsBedrock/AwsBedrockConverseFormatConverter.php b/src/Api/Providers/AwsBedrock/AwsBedrockConverseFormatConverter.php index 8923d50..f76bd2f 100644 --- a/src/Api/Providers/AwsBedrock/AwsBedrockConverseFormatConverter.php +++ b/src/Api/Providers/AwsBedrock/AwsBedrockConverseFormatConverter.php @@ -124,6 +124,9 @@ public function getIterator(): Generator break; case 'contentBlockStop': case 'metadata': + if (isset($event['usage'])) { + yield $this->formatUsageEvent($created, $event['usage']); + } break; case 'messageStop': yield $this->formatMessageStopEvent($created, $event['stopReason'] ?? 'stop'); @@ -152,6 +155,32 @@ public function getModel(): string return $this->model; } + private function formatUsageEvent(int $created, array $usage): string + { + return $this->formatOpenAiEvent([ + 'id' => $this->messageId ?? ('bedrock-' . uniqid()), + 'object' => 'chat.completion.chunk', + 'created' => $created, + 'model' => $this->model ?: 'aws.bedrock', + 'choices' => null, + 'usage' => [ + 'prompt_tokens' => $usage['inputTokens'] ?? 0, + 'completion_tokens' => $usage['outputTokens'] ?? 0, + 'total_tokens' => $usage['totalTokens'] ?? 0, + 'prompt_tokens_details' => [ + 'cache_write_input_tokens' => $usage['cacheWriteInputTokens'] ?? 0, + 'cache_read_input_tokens' => $usage['cacheReadInputTokens'] ?? 0, + // 兼容旧参数 + 'audio_tokens' => 0, + 'cached_tokens' => $usage['cacheWriteInputTokens'] ?? 0, + ], + 'completion_tokens_details' => [ + 'reasoning_tokens' => 0, + ], + ], + ]); + } + /** * 格式化消息开始事件. * diff --git a/src/Api/Response/ChatCompletionStreamResponse.php b/src/Api/Response/ChatCompletionStreamResponse.php index fe43bf5..9ee7536 100644 --- a/src/Api/Response/ChatCompletionStreamResponse.php +++ b/src/Api/Response/ChatCompletionStreamResponse.php @@ -347,6 +347,10 @@ private function iterateWithLegacyMethod(): Generator */ private function handleStreamCompletion(float $startTime): void { + if (! isset($this->afterChatCompletionsStreamEvent)) { + return; + } + // Set duration and create completion response $this->afterChatCompletionsStreamEvent->setDuration(microtime(true) - $startTime); From ba698184773f0934bbfa696022bab3ff57193a22 Mon Sep 17 00:00:00 2001 From: lihq1403 Date: Mon, 26 May 2025 14:48:23 +0800 Subject: [PATCH 04/10] feat: Remove debug output from event dispatching in EventUtil --- src/Utils/EventUtil.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Utils/EventUtil.php b/src/Utils/EventUtil.php index 7fd213c..2349e2b 100644 --- a/src/Utils/EventUtil.php +++ b/src/Utils/EventUtil.php @@ -19,7 +19,6 @@ class EventUtil { public static function dispatch(object $event): void { - var_dump($event); $container = ApplicationContext::getContainer(); if (! $container->has(EventDispatcherInterface::class)) { return; From 17b51a358ec6b7f00e9f5943e4a165e229ffec89 Mon Sep 17 00:00:00 2001 From: lihq1403 Date: Mon, 26 May 2025 16:02:47 +0800 Subject: [PATCH 05/10] feat: Refactor AfterChatCompletionsEvent to include getters and setters, and remove large objects from completion response --- src/Api/Response/AbstractResponse.php | 5 ++ src/Event/AfterChatCompletionsEvent.php | 54 +++++++++++++++++-- src/Event/AfterChatCompletionsStreamEvent.php | 40 ++------------ 3 files changed, 59 insertions(+), 40 deletions(-) diff --git a/src/Api/Response/AbstractResponse.php b/src/Api/Response/AbstractResponse.php index c21b010..411d962 100644 --- a/src/Api/Response/AbstractResponse.php +++ b/src/Api/Response/AbstractResponse.php @@ -84,5 +84,10 @@ public function setUsage(?Usage $usage): self return $this; } + public function removeBigObject(): void + { + unset($this->originResponse, $this->logger); + } + abstract protected function parseContent(): self; } diff --git a/src/Event/AfterChatCompletionsEvent.php b/src/Event/AfterChatCompletionsEvent.php index 23b339a..61baca3 100644 --- a/src/Event/AfterChatCompletionsEvent.php +++ b/src/Event/AfterChatCompletionsEvent.php @@ -17,9 +17,55 @@ class AfterChatCompletionsEvent { + public ChatCompletionRequest $completionRequest; + + public ChatCompletionResponse $completionResponse; + + public float $duration; + public function __construct( - public ChatCompletionRequest $completionRequest, - public ChatCompletionResponse $completionResponse, - public float $duration - ) {} + ChatCompletionRequest $completionRequest, + ?ChatCompletionResponse $completionResponse, + float $duration + ) { + $this->completionRequest = $completionRequest; + $this->setCompletionResponse($completionResponse); + $this->duration = $duration; + } + + public function getCompletionRequest(): ChatCompletionRequest + { + return $this->completionRequest; + } + + public function setCompletionRequest(ChatCompletionRequest $completionRequest): void + { + $this->completionRequest = $completionRequest; + } + + public function getCompletionResponse(): ChatCompletionResponse + { + return $this->completionResponse; + } + + public function setCompletionResponse(?ChatCompletionResponse $completionResponse): void + { + if (! $completionResponse) { + return; + } + // 移除大对象属性 + $completionResponse = clone $completionResponse; + $completionResponse->removeBigObject(); + $this->completionResponse = $completionResponse; + } + + public function getDuration(): float + { + return $this->duration; + } + + public function setDuration(float $duration): void + { + $this->duration = $duration; + } } diff --git a/src/Event/AfterChatCompletionsStreamEvent.php b/src/Event/AfterChatCompletionsStreamEvent.php index 0130bd5..1ac1efb 100644 --- a/src/Event/AfterChatCompletionsStreamEvent.php +++ b/src/Event/AfterChatCompletionsStreamEvent.php @@ -13,53 +13,21 @@ namespace Hyperf\Odin\Event; use Hyperf\Odin\Api\Request\ChatCompletionRequest; -use Hyperf\Odin\Api\Response\ChatCompletionResponse; -class AfterChatCompletionsStreamEvent +class AfterChatCompletionsStreamEvent extends AfterChatCompletionsEvent { - public ChatCompletionRequest $completionRequest; - - public ChatCompletionResponse $completionResponse; - - public float $duration; - public float $firstResponseDuration; public function __construct( ChatCompletionRequest $completionRequest, float $firstResponseDuration, ) { - $this->completionRequest = $completionRequest; $this->firstResponseDuration = $firstResponseDuration; + parent::__construct($completionRequest, null, 0); } - public function getCompletionRequest(): ChatCompletionRequest - { - return $this->completionRequest; - } - - public function setCompletionRequest(ChatCompletionRequest $completionRequest): void - { - $this->completionRequest = $completionRequest; - } - - public function getCompletionResponse(): ChatCompletionResponse - { - return $this->completionResponse; - } - - public function setCompletionResponse(ChatCompletionResponse $completionResponse): void - { - $this->completionResponse = $completionResponse; - } - - public function getDuration(): float - { - return $this->duration; - } - - public function setDuration(float $duration): void + public function getFirstResponseDuration(): float { - $this->duration = $duration; + return $this->firstResponseDuration; } } From 09b3a6dc46280d8b9020ecfbbdb06072b72098af Mon Sep 17 00:00:00 2001 From: lihq1403 Date: Mon, 26 May 2025 16:34:30 +0800 Subject: [PATCH 06/10] feat: Optimize token estimation in ChatCompletionRequest and ChatCompletionResponse --- src/Api/Request/ChatCompletionRequest.php | 5 ++++- src/Api/Response/ChatCompletionResponse.php | 15 +++++++++++++++ src/Utils/TokenEstimator.php | 2 +- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/Api/Request/ChatCompletionRequest.php b/src/Api/Request/ChatCompletionRequest.php index 952b7e2..db745ec 100644 --- a/src/Api/Request/ChatCompletionRequest.php +++ b/src/Api/Request/ChatCompletionRequest.php @@ -125,7 +125,10 @@ public function createOptions(): array */ public function calculateTokenEstimates(): int { - $estimator = new TokenEstimator($model ?? $this->model); + if ($this->totalTokenEstimate) { + return $this->totalTokenEstimate; + } + $estimator = new TokenEstimator($this->model); $totalTokens = 0; // 为每个消息计算token diff --git a/src/Api/Response/ChatCompletionResponse.php b/src/Api/Response/ChatCompletionResponse.php index 30fedf8..dd5b42f 100644 --- a/src/Api/Response/ChatCompletionResponse.php +++ b/src/Api/Response/ChatCompletionResponse.php @@ -13,6 +13,7 @@ namespace Hyperf\Odin\Api\Response; use Hyperf\Odin\Exception\LLMException\LLMApiException; +use Hyperf\Odin\Utils\TokenEstimator; use Stringable; class ChatCompletionResponse extends AbstractResponse implements Stringable @@ -30,6 +31,8 @@ class ChatCompletionResponse extends AbstractResponse implements Stringable */ protected ?array $choices = []; + private ?int $totalTokenEstimate = null; + public function __toString(): string { return trim($this->getChoices()[0]?->getMessage()?->getContent() ?: ''); @@ -98,6 +101,18 @@ public function setChoices(?array $choices): self return $this; } + public function calculateTokenEstimates(): int + { + if ($this->totalTokenEstimate) { + return $this->totalTokenEstimate; + } + $estimator = new TokenEstimator($this->model); + + $this->totalTokenEstimate = $estimator->estimateTokens($this->getFirstChoice()?->getMessage()?->getContent() ?? ''); + + return $this->totalTokenEstimate; + } + protected function parseContent(): self { $this->content = $this->originResponse->getBody()->getContents(); diff --git a/src/Utils/TokenEstimator.php b/src/Utils/TokenEstimator.php index 07bbf0a..9e4e785 100644 --- a/src/Utils/TokenEstimator.php +++ b/src/Utils/TokenEstimator.php @@ -86,7 +86,7 @@ public function __construct( */ public function estimateTokens(string $text): int { - if (empty($text)) { + if ($text === '') { return 0; } From fd4eb35fe234cf70beda6e8dd1c721e785262a27 Mon Sep 17 00:00:00 2001 From: lihq1403 Date: Mon, 26 May 2025 17:08:17 +0800 Subject: [PATCH 07/10] feat: Add business parameters support to embeddings and dispatch AfterEmbeddingsEvent --- src/Api/Providers/AbstractClient.php | 3 ++ src/Api/Request/EmbeddingRequest.php | 39 ++++++++++++++-- src/Contract/Model/EmbeddingInterface.php | 2 +- src/Event/AfterEmbeddingsEvent.php | 57 +++++++++++++++++++++++ src/Model/AbstractModel.php | 4 +- src/Model/QianFanModel.php | 4 +- 6 files changed, 101 insertions(+), 8 deletions(-) create mode 100644 src/Event/AfterEmbeddingsEvent.php diff --git a/src/Api/Providers/AbstractClient.php b/src/Api/Providers/AbstractClient.php index 7e8ce10..e0dd9a5 100644 --- a/src/Api/Providers/AbstractClient.php +++ b/src/Api/Providers/AbstractClient.php @@ -27,6 +27,7 @@ use Hyperf\Odin\Contract\Api\ConfigInterface; use Hyperf\Odin\Event\AfterChatCompletionsEvent; use Hyperf\Odin\Event\AfterChatCompletionsStreamEvent; +use Hyperf\Odin\Event\AfterEmbeddingsEvent; use Hyperf\Odin\Exception\LLMException; use Hyperf\Odin\Exception\LLMException\ErrorHandlerInterface; use Hyperf\Odin\Exception\LLMException\ErrorMappingManager; @@ -170,6 +171,8 @@ public function embeddings(EmbeddingRequest $embeddingRequest): EmbeddingRespons 'data' => $embeddingResponse->toArray(), ]); + EventUtil::dispatch(new AfterEmbeddingsEvent($embeddingRequest, $embeddingResponse, $duration)); + return $embeddingResponse; } catch (Throwable $e) { throw $this->convertException($e, [ diff --git a/src/Api/Request/EmbeddingRequest.php b/src/Api/Request/EmbeddingRequest.php index c06a256..2ef675f 100644 --- a/src/Api/Request/EmbeddingRequest.php +++ b/src/Api/Request/EmbeddingRequest.php @@ -18,6 +18,10 @@ class EmbeddingRequest implements RequestInterface { + private array $businessParams = []; + + private bool $includeBusinessParams = false; + /** * @param string|string[] $input 需要嵌入的文本，可以是字符串或字符串数组 * @param string $model 使用的嵌入模型ID @@ -53,12 +57,17 @@ public function createOptions(): array { $this->validate(); + $body = [ + 'model' => $this->model, + 'input' => $this->input, + 'encoding_format' => $this->encoding_format, + ]; + if ($this->includeBusinessParams && ! empty($this->businessParams)) { + $body['business_params'] = $this->businessParams; + } + $options = [ - RequestOptions::JSON => [ - 'input' => $this->input, - 'model' => $this->model, - 'encoding_format' => $this->encoding_format, - ], + RequestOptions::JSON => $body, ]; if ($this->user !== null) { @@ -111,4 +120,24 @@ public function getDimensions(): ?array { return $this->dimensions; } + + public function getBusinessParams(): array + { + return $this->businessParams; + } + + public function setBusinessParams(array $businessParams): void + { + $this->businessParams = $businessParams; + } + + public function isIncludeBusinessParams(): bool + { + return $this->includeBusinessParams; + } + + public function setIncludeBusinessParams(bool $includeBusinessParams): void + { + $this->includeBusinessParams = $includeBusinessParams; + } } diff --git a/src/Contract/Model/EmbeddingInterface.php b/src/Contract/Model/EmbeddingInterface.php index cc26c66..b195491 100644 --- a/src/Contract/Model/EmbeddingInterface.php +++ b/src/Contract/Model/EmbeddingInterface.php @@ -19,7 +19,7 @@ interface EmbeddingInterface { public function embedding(string $input): Embedding; - public function embeddings(array|string $input, ?string $encoding_format = 'float', ?string $user = null): EmbeddingResponse; + public function embeddings(array|string $input, ?string $encoding_format = 'float', ?string $user = null, array $businessParams = []): EmbeddingResponse; public function getModelName(): string; diff --git a/src/Event/AfterEmbeddingsEvent.php b/src/Event/AfterEmbeddingsEvent.php new file mode 100644 index 0000000..92179cb --- /dev/null +++ b/src/Event/AfterEmbeddingsEvent.php @@ -0,0 +1,57 @@ +embeddingRequest = $embeddingRequest; + $this->setEmbeddingResponse($embeddingResponse); + $this->duration = $duration; + } + + public function getEmbeddingRequest(): EmbeddingRequest + { + return $this->embeddingRequest; + } + + public function getEmbeddingResponse(): EmbeddingResponse + { + return $this->embeddingResponse; + } + + public function getDuration(): float + { + return $this->duration; + } + + public function setEmbeddingResponse(EmbeddingResponse $embeddingResponse): void + { + $embeddingResponse = clone $embeddingResponse; + $embeddingResponse->removeBigObject(); + $this->embeddingResponse = $embeddingResponse; + } +} diff --git a/src/Model/AbstractModel.php b/src/Model/AbstractModel.php index 33a17bf..25534e4 100644 --- a/src/Model/AbstractModel.php +++ b/src/Model/AbstractModel.php @@ -198,7 +198,7 @@ public function embedding(array|string $input, ?string $encoding_format = 'float return new Embedding($embeddings[0]); } - public function embeddings(array|string $input, ?string $encoding_format = 'float', ?string $user = null): EmbeddingResponse + public function embeddings(array|string $input, ?string $encoding_format = 'float', ?string $user = null, array $businessParams = []): EmbeddingResponse { try { // 检查模型是否支持嵌入功能 @@ -209,6 +209,8 @@ public function embeddings(array|string $input, ?string $encoding_format = 'floa input: $input, model: $this->model ); + $embeddingRequest->setBusinessParams($businessParams); + $embeddingRequest->setIncludeBusinessParams($this->includeBusinessParams); return $client->embeddings($embeddingRequest); } catch (Throwable $e) { diff --git a/src/Model/QianFanModel.php b/src/Model/QianFanModel.php index 751c35c..9de50f1 100644 --- a/src/Model/QianFanModel.php +++ b/src/Model/QianFanModel.php @@ -22,7 +22,7 @@ class QianFanModel extends AbstractModel { protected bool $streamIncludeUsage = true; - public function embeddings(array|string $input, ?string $encoding_format = 'float', ?string $user = null): EmbeddingResponse + public function embeddings(array|string $input, ?string $encoding_format = 'float', ?string $user = null, array $businessParams = []): EmbeddingResponse { try { // 检查模型是否支持嵌入功能 @@ -37,6 +37,8 @@ public function embeddings(array|string $input, ?string $encoding_format = 'floa input: $input, model: $this->model ); + $embeddingRequest->setBusinessParams($businessParams); + $embeddingRequest->setIncludeBusinessParams($this->includeBusinessParams); return $client->embeddings($embeddingRequest); } catch (Throwable $e) { From 3c8194f7b00080756f0af8d35c9248206920277b Mon Sep 17 00:00:00 2001 From: lihq1403 Date: Mon, 26 May 2025 17:18:14 +0800 Subject: [PATCH 08/10] feat: Refactor ModelMapper to use ModelType constants for improved readability and maintainability --- src/Constants/ModelType.php | 29 +++++++++++++++++++++++++++++ src/ModelMapper.php | 29 +++++++++++++++-------------- 2 files changed, 44 insertions(+), 14 deletions(-) create mode 100644 src/Constants/ModelType.php diff --git a/src/Constants/ModelType.php b/src/Constants/ModelType.php new file mode 100644 index 0000000..1d68027 --- /dev/null +++ b/src/Constants/ModelType.php @@ -0,0 +1,29 @@ +models['chat'][$model])) { - return $this->models['chat'][$model]; + if (isset($this->models[ModelType::CHAT][$model])) { + return $this->models[ModelType::CHAT][$model]; } // 如果模型未缓存，创建模型 @@ -102,11 +103,11 @@ public function getChatModel(string $model): ModelInterface $this->addModel($model, $modelConfig); - if (! isset($this->models['chat'][$model])) { + if (! isset($this->models[ModelType::CHAT][$model])) { throw new InvalidArgumentException(sprintf('Failed to create Chat Model %s.', $model)); } - return $this->models['chat'][$model]; + return $this->models[ModelType::CHAT][$model]; } /** @@ -119,8 +120,8 @@ public function getEmbeddingModel(string $model): EmbeddingInterface } // 检查缓存 - if (isset($this->models['embedding'][$model])) { - return $this->models['embedding'][$model]; + if (isset($this->models[ModelType::EMBEDDING][$model])) { + return $this->models[ModelType::EMBEDDING][$model]; } // 如果模型未缓存，创建模型 @@ -131,11 +132,11 @@ public function getEmbeddingModel(string $model): EmbeddingInterface $this->addModel($model, $modelConfig); - if (! isset($this->models['embedding'][$model])) { + if (! isset($this->models[ModelType::EMBEDDING][$model])) { throw new InvalidArgumentException(sprintf('Failed to create Embedding Model %s.', $model)); } - return $this->models['embedding'][$model]; + return $this->models[ModelType::EMBEDDING][$model]; } /** @@ -143,11 +144,11 @@ public function getEmbeddingModel(string $model): EmbeddingInterface */ public function getModels(string $type = ''): array { - if ($type === 'embedding') { - return $this->models['embedding'] ?? []; + if ($type === ModelType::EMBEDDING) { + return $this->models[ModelType::EMBEDDING] ?? []; } - if ($type === 'chat') { - return $this->models['chat'] ?? []; + if ($type === ModelType::CHAT) { + return $this->models[ModelType::CHAT] ?? []; } return $this->models; } @@ -192,10 +193,10 @@ public function addModel(string $model, array $item): void // 根据模型类型缓存实例 if ($modelOptions->isEmbedding() && $modelObject instanceof EmbeddingInterface) { - $this->models['embedding'][$model] = $modelObject; + $this->models[ModelType::EMBEDDING][$model] = $modelObject; } if ($modelOptions->isChat() && $modelObject instanceof ModelInterface) { - $this->models['chat'][$model] = $modelObject; + $this->models[ModelType::CHAT][$model] = $modelObject; } } } From 137fef7f1ddca9faf046faadf2cee3aeede965e5 Mon Sep 17 00:00:00 2001 From: lihq1403 Date: Mon, 26 May 2025 17:39:05 +0800 Subject: [PATCH 09/10] feat: Add token estimation functionality to EmbeddingRequest --- src/Api/Request/EmbeddingRequest.php | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/Api/Request/EmbeddingRequest.php b/src/Api/Request/EmbeddingRequest.php index 2ef675f..89e3dc1 100644 --- a/src/Api/Request/EmbeddingRequest.php +++ b/src/Api/Request/EmbeddingRequest.php @@ -15,6 +15,7 @@ use GuzzleHttp\RequestOptions; use Hyperf\Odin\Contract\Api\Request\RequestInterface; use Hyperf\Odin\Exception\InvalidArgumentException; +use Hyperf\Odin\Utils\TokenEstimator; class EmbeddingRequest implements RequestInterface { @@ -22,6 +23,8 @@ class EmbeddingRequest implements RequestInterface private bool $includeBusinessParams = false; + private ?int $totalTokenEstimate = null; + /** * @param string|string[] $input 需要嵌入的文本，可以是字符串或字符串数组 * @param string $model 使用的嵌入模型ID @@ -140,4 +143,25 @@ public function setIncludeBusinessParams(bool $includeBusinessParams): void { $this->includeBusinessParams = $includeBusinessParams; } + + public function calculateTokenEstimates(): int + { + if ($this->totalTokenEstimate) { + return $this->totalTokenEstimate; + } + $estimator = new TokenEstimator($this->model); + + $input = $this->input; + if (! is_array($input)) { + $input = [$input]; + } + $totalTokens = 0; + foreach ($input as $item) { + // 估算每个输入的token数量 + $totalTokens += $estimator->estimateTokens($item); + } + $this->totalTokenEstimate = $totalTokens; + + return $this->totalTokenEstimate; + } } From ff2b9d51045f0868b9eeeedc1d47184296582a2c Mon Sep 17 00:00:00 2001 From: lihq1403 Date: Mon, 26 May 2025 17:41:24 +0800 Subject: [PATCH 10/10] feat: Add token estimation functionality to EmbeddingRequest --- src/Api/Request/EmbeddingRequest.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Api/Request/EmbeddingRequest.php b/src/Api/Request/EmbeddingRequest.php index 89e3dc1..c919094 100644 --- a/src/Api/Request/EmbeddingRequest.php +++ b/src/Api/Request/EmbeddingRequest.php @@ -144,6 +144,11 @@ public function setIncludeBusinessParams(bool $includeBusinessParams): void $this->includeBusinessParams = $includeBusinessParams; } + public function getTotalTokenEstimate(): ?int + { + return $this->totalTokenEstimate; + } + public function calculateTokenEstimates(): int { if ($this->totalTokenEstimate) {