diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 49e385f..f258e83 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -45,3 +45,33 @@ jobs: - name: Run phpstan run: vendor/bin/phpstan analyse -c phpstan.dist.neon --no-progress --error-format=github + + type-coverage: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: 8.3 + extensions: dom, curl, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick, fileinfo + + - name: Get Composer Cache Directory + id: composer-cache + run: echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT + + - name: Cache Composer dependencies + uses: actions/cache@v4 + with: + path: ${{ steps.composer-cache.outputs.dir }} + key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }} + restore-keys: | + ${{ runner.os }}-composer- + + - name: Install dependencies + run: composer install --prefer-dist --no-interaction --no-progress + + - name: Check type coverage + run: vendor/bin/pest --type-coverage --min=100 diff --git a/src/LLM/AbstractLLM.php b/src/LLM/AbstractLLM.php index 8ae74e6..f110499 100644 --- a/src/LLM/AbstractLLM.php +++ b/src/LLM/AbstractLLM.php @@ -18,10 +18,12 @@ use Cortex\Contracts\OutputParser; use Cortex\Support\Traits\CanPipe; use Cortex\Exceptions\LLMException; +use Cortex\LLM\Data\ChatGeneration; use Cortex\JsonSchema\SchemaFactory; use Cortex\ModelInfo\Data\ModelInfo; use Cortex\LLM\Data\ChatStreamResult; use Cortex\Exceptions\PipelineException; +use Cortex\LLM\Data\ChatGenerationChunk; use Cortex\ModelInfo\Enums\ModelFeature; use Cortex\JsonSchema\Types\ObjectSchema; use Cortex\LLM\Data\Messages\UserMessage; @@ -50,6 +52,8 @@ abstract class AbstractLLM implements LLM protected ?StructuredOutputConfig $structuredOutputConfig = null; + protected StructuredOutputMode $structuredOutputMode = StructuredOutputMode::Auto; + protected ?OutputParser $outputParser = null; protected ?string $outputParserError = null; @@ -64,6 +68,8 @@ abstract class AbstractLLM implements LLM protected bool $shouldApplyFormatInstructions = false; + protected bool $shouldParseOutput = true; + /** * @var array<\Cortex\ModelInfo\Enums\ModelFeature> */ @@ -116,8 +122,11 @@ public function output(OutputParser $parser): Pipeline /** * @param array $tools */ - public function withTools(array $tools, ToolChoice|string $toolChoice = ToolChoice::Auto): static - { + public function withTools( + array $tools, + ToolChoice|string $toolChoice = ToolChoice::Auto, + bool $allowParallelToolCalls = true, + ): static { $this->supportsFeatureOrFail(ModelFeature::ToolCalling); $this->toolConfig = $tools === [] @@ -125,6 +134,7 @@ public function withTools(array $tools, ToolChoice|string $toolChoice = ToolChoi : new ToolConfig( Utils::toToolCollection($tools)->all(), $toolChoice, + $allowParallelToolCalls, ); return $this; @@ -133,9 +143,15 @@ public function withTools(array $tools, ToolChoice|string $toolChoice = ToolChoi /** * Add a tool to the LLM. */ - public function addTool(Tool|Closure|string $tool, ToolChoice|string $toolChoice = ToolChoice::Auto): static - { - return $this->withTools([...($this->toolConfig->tools ?? []), $tool], $toolChoice); + public function addTool( + Tool|Closure|string $tool, + ToolChoice|string $toolChoice = ToolChoice::Auto, + bool $allowParallelToolCalls = true, + ): static { + return $this->withTools([ + ...($this->toolConfig->tools ?? []), + $tool, + ], $toolChoice, $allowParallelToolCalls); } /** @@ -166,6 +182,7 @@ public function withStructuredOutput( bool $strict = true, StructuredOutputMode $outputMode = StructuredOutputMode::Auto, ): static { + $this->structuredOutputMode = $outputMode; [$schema, $outputParser] = $this->resolveSchemaAndOutputParser($output, $strict); $this->withOutputParser($outputParser); @@ -386,6 +403,64 @@ protected static function applyFormatInstructions( return $messages; } + public function shouldParseOutput(bool $shouldParseOutput = true): static + { + $this->shouldParseOutput = $shouldParseOutput; + + return $this; + } + + protected function applyOutputParserIfApplicable( + ChatGeneration|ChatGenerationChunk $generationOrChunk, + ): ChatGeneration|ChatGenerationChunk { + if ($this->shouldParseOutput && $this->outputParser !== null) { + try { + $parsedOutput = $this->outputParser->parse($generationOrChunk); + $generationOrChunk = $generationOrChunk->cloneWithParsedOutput($parsedOutput); + } catch (OutputParserException $e) { + $this->outputParserError = $e->getMessage(); + } + } + + return $generationOrChunk; + } + + /** + * Apply the format instructions to the messages if applicable. + * + * @return \Cortex\LLM\Data\Messages\MessageCollection + */ + protected function applyFormatInstructionsIfApplicable( + MessageCollection $messages, + ): MessageCollection { + if ($this->shouldApplyFormatInstructions && $formatInstructions = $this->outputParser?->formatInstructions()) { + return static::applyFormatInstructions($messages, $formatInstructions); + } + + return $messages; + } + + /** + * Resolve the messages to be used for the LLM. + * + * @param \Cortex\LLM\Data\Messages\MessageCollection|\Cortex\LLM\Contracts\Message|non-empty-array|string $messages + * + * @throws \Cortex\Exceptions\LLMException If no messages are provided + * + * @return \Cortex\LLM\Data\Messages\MessageCollection + */ + protected function resolveMessages( + MessageCollection|Message|array|string $messages, + ): MessageCollection { + $messages = Utils::toMessageCollection($messages)->withoutPlaceholders(); + + if ($messages->isEmpty()) { + throw new LLMException('You must provide at least one message to the LLM.'); + } + + return $this->applyFormatInstructionsIfApplicable($messages); + } + /** * Resolve the schema and output parser from the given output type. * diff --git a/src/LLM/CacheDecorator.php b/src/LLM/CacheDecorator.php index 1a15f00..5c80b28 100644 --- a/src/LLM/CacheDecorator.php +++ b/src/LLM/CacheDecorator.php @@ -128,6 +128,13 @@ public function ignoreFeatures(bool $ignoreModelFeatures = true): static return $this; } + public function shouldParseOutput(bool $shouldParseOutput = true): static + { + $this->llm = $this->llm->shouldParseOutput($shouldParseOutput); + + return $this; + } + public function withModel(string $model): static { $this->llm = $this->llm->withModel($model); diff --git a/src/LLM/Contracts/LLM.php b/src/LLM/Contracts/LLM.php index c618235..805a20e 100644 --- a/src/LLM/Contracts/LLM.php +++ b/src/LLM/Contracts/LLM.php @@ -150,4 +150,11 @@ public function getFeatures(): array; * Get the model info for the LLM. */ public function getModelInfo(): ?ModelInfo; + + /** + * Set whether the output should be parsed. + * This may be set to false when called in a pipeline context and output parsing + * is done as part of the next pipeable. + */ + public function shouldParseOutput(bool $shouldParseOutput = true): static; } diff --git a/src/LLM/Contracts/Tool.php b/src/LLM/Contracts/Tool.php index 51864c0..ec07b16 100644 --- a/src/LLM/Contracts/Tool.php +++ b/src/LLM/Contracts/Tool.php @@ -5,7 +5,7 @@ namespace Cortex\LLM\Contracts; use Cortex\LLM\Data\ToolCall; -use Cortex\JsonSchema\Contracts\Schema; +use Cortex\JsonSchema\Types\ObjectSchema; use Cortex\LLM\Data\Messages\ToolMessage; interface Tool @@ -23,7 +23,7 @@ public function description(): string; /** * Get the schema of the tool. */ - public function schema(): Schema; + public function schema(): ObjectSchema; /** * Get the formatted output of the tool. diff --git a/src/LLM/Data/Messages/AssistantMessage.php b/src/LLM/Data/Messages/AssistantMessage.php index ba18565..079eb79 100644 --- a/src/LLM/Data/Messages/AssistantMessage.php +++ b/src/LLM/Data/Messages/AssistantMessage.php @@ -27,6 +27,7 @@ public function __construct( public ?ToolCallCollection $toolCalls = null, public ?string $name = null, public ?ResponseMetadata $metadata = null, + public ?string $id = null, ) { $this->role = MessageRole::Assistant; } diff --git a/src/LLM/Data/Messages/Content/ReasoningContent.php b/src/LLM/Data/Messages/Content/ReasoningContent.php index 9e1047c..e53a6ad 100644 --- a/src/LLM/Data/Messages/Content/ReasoningContent.php +++ b/src/LLM/Data/Messages/Content/ReasoningContent.php @@ -7,6 +7,7 @@ readonly class ReasoningContent extends AbstractContent { public function __construct( + public string $id, public string $reasoning, ) {} } diff --git a/src/LLM/Data/Messages/Content/TextContent.php b/src/LLM/Data/Messages/Content/TextContent.php index 8b542d5..e84f1f4 100644 --- a/src/LLM/Data/Messages/Content/TextContent.php +++ b/src/LLM/Data/Messages/Content/TextContent.php @@ -10,18 +10,18 @@ readonly class TextContent extends AbstractContent { public function __construct( - public string $text, + public ?string $text = null, ) {} #[Override] public function variables(): array { - return Utils::findVariables($this->text); + return Utils::findVariables($this->text ?? ''); } #[Override] public function replaceVariables(array $variables): self { - return new self(Utils::replaceVariables($this->text, $variables)); + return new self(Utils::replaceVariables($this->text ?? '', $variables)); } } diff --git a/src/LLM/Data/ToolConfig.php b/src/LLM/Data/ToolConfig.php index dd7d3ed..5bf97e8 100644 --- a/src/LLM/Data/ToolConfig.php +++ b/src/LLM/Data/ToolConfig.php @@ -14,5 +14,6 @@ public function __construct( public array $tools, public ToolChoice|string $toolChoice = ToolChoice::Auto, + public bool $allowParallelToolCalls = true, ) {} } diff --git a/src/LLM/Data/Usage.php b/src/LLM/Data/Usage.php index 64feb6e..5c9f6cb 100644 --- a/src/LLM/Data/Usage.php +++ b/src/LLM/Data/Usage.php @@ -17,6 +17,7 @@ public function __construct( public int $promptTokens, public ?int $completionTokens = null, public ?int $cachedTokens = null, + public ?int $reasoningTokens = null, ?int $totalTokens = null, public ?float $inputCost = null, public ?float $outputCost = null, diff --git a/src/LLM/Drivers/AnthropicChat.php b/src/LLM/Drivers/AnthropicChat.php index 01d1edb..0d17fb7 100644 --- a/src/LLM/Drivers/AnthropicChat.php +++ b/src/LLM/Drivers/AnthropicChat.php @@ -6,11 +6,12 @@ use Generator; use Throwable; +use JsonException; use DateTimeImmutable; -use Cortex\Support\Utils; use Cortex\LLM\Data\Usage; use Cortex\LLM\AbstractLLM; use Illuminate\Support\Arr; +use Cortex\LLM\Data\ToolCall; use Cortex\LLM\Contracts\Tool; use Cortex\Events\ChatModelEnd; use Cortex\LLM\Data\ChatResult; @@ -19,26 +20,28 @@ use Cortex\Events\ChatModelError; use Cortex\Events\ChatModelStart; use Cortex\LLM\Contracts\Message; +use Cortex\LLM\Data\FunctionCall; use Cortex\LLM\Enums\MessageRole; +use Cortex\Events\ChatModelStream; use Cortex\LLM\Enums\FinishReason; use Cortex\Exceptions\LLMException; use Cortex\LLM\Data\ChatGeneration; use Cortex\LLM\Data\ChatStreamResult; use Cortex\LLM\Data\ResponseMetadata; use Anthropic\Contracts\ClientContract; +use Cortex\LLM\Data\ToolCallCollection; use Cortex\LLM\Data\ChatGenerationChunk; -use Cortex\JsonSchema\Types\ObjectSchema; use Cortex\LLM\Data\Messages\ToolMessage; use Cortex\ModelInfo\Enums\ModelProvider; use Cortex\LLM\Data\Messages\SystemMessage; +use Cortex\Tasks\Enums\StructuredOutputMode; use Cortex\LLM\Data\Messages\AssistantMessage; use Cortex\LLM\Data\Messages\MessageCollection; use Anthropic\Responses\Messages\CreateResponse; use Anthropic\Responses\Messages\StreamResponse; use Cortex\LLM\Data\Messages\Content\TextContent; -use Cortex\LLM\Data\Messages\Content\ImageContent; use Anthropic\Responses\Messages\CreateResponseUsage; -use Cortex\LLM\Data\Messages\Content\DocumentContent; +use Cortex\LLM\Data\Messages\Content\ReasoningContent; use Anthropic\Responses\Messages\CreateResponseContent; use Anthropic\Responses\Messages\CreateStreamedResponseUsage; use Anthropic\Testing\Responses\Fixtures\Messages\CreateResponseFixture; @@ -57,16 +60,7 @@ public function invoke( MessageCollection|Message|array|string $messages, array $additionalParameters = [], ): ChatResult|ChatStreamResult { - $messages = Utils::toMessageCollection($messages)->withoutPlaceholders(); - - if ($messages->isEmpty()) { - throw new LLMException('You must provide at least one message to the LLM.'); - } - - // Apply format instructions if applicable. - if ($this->shouldApplyFormatInstructions && $formatInstructions = $this->outputParser?->formatInstructions()) { - $messages = static::applyFormatInstructions($messages, $formatInstructions); - } + $messages = $this->resolveMessages($messages); [$systemMessages, $messages] = $messages->partition( fn(Message $message): bool => $message instanceof SystemMessage, @@ -106,44 +100,63 @@ public function invoke( */ protected function mapResponse(CreateResponse $response): ChatResult { - $content = $response->content[0]; - // $toolCalls = $content->toolCalls === [] ? null : new ToolCallCollection( - // collect($content->toolCalls) - // ->map(fn(CreateResponseToolCall $toolCall): ToolCall => new ToolCall( - // $toolCall->id, - // new FunctionCall( - // $toolCall->function->name, - // json_decode($toolCall->function->arguments, true, flags: JSON_THROW_ON_ERROR), - // ), - // )) - // ->values() - // ->all(), - // ); + $toolCalls = array_filter( + $response->content, + fn(CreateResponseContent $content): bool => $content->type === 'tool_use', + ); + + $toolCalls = collect($toolCalls) + ->map(fn(CreateResponseContent $content): ToolCall => new ToolCall( + $content->id, + new FunctionCall($content->name, $content->input ?? []), + )) + ->values() + ->all(); + + $toolCalls = $toolCalls !== [] + ? new ToolCallCollection($toolCalls) + : null; $usage = $this->mapUsage($response->usage); $finishReason = static::mapFinishReason($response->stop_reason ?? null); - $generations = collect($response->content) - ->map(fn(CreateResponseContent $content): ChatGeneration => new ChatGeneration( - message: new AssistantMessage( - content: $content->text, - // toolCalls: $toolCalls, - metadata: new ResponseMetadata( - id: $response->id, - model: $response->model, - provider: $this->modelProvider, - finishReason: $finishReason, - usage: $usage, - ), - ), - index: 0, - createdAt: new DateTimeImmutable(), - finishReason: $finishReason, - )) + $contents = collect($response->content) + ->map(function (CreateResponseContent $content): TextContent|ReasoningContent|null { + return match ($content->type) { + 'text' => new TextContent($content->text), + // TODO: Use different anthropic client, since current one seems abandoned. + 'thinking' => $content->id !== null ? new ReasoningContent( + $content->id, + $content->text ?? '', + ) : null, + // 'tool_use' => new ToolUseContent($content->tool_use), + default => null, + }; + }) + ->filter() ->all(); + $generation = new ChatGeneration( + message: new AssistantMessage( + content: $contents, + toolCalls: $toolCalls, + metadata: new ResponseMetadata( + id: $response->id, + model: $response->model, + provider: $this->modelProvider, + finishReason: $finishReason, + usage: $usage, + ), + ), + index: 0, + createdAt: new DateTimeImmutable(), + finishReason: $finishReason, + ); + + $generation = $this->applyOutputParserIfApplicable($generation); + $result = new ChatResult( - $generations, + [$generation], $usage, $response->toArray(), // @phpstan-ignore argument.type ); @@ -165,94 +178,129 @@ protected function mapStreamResponse(StreamResponse $response): ChatStreamResult return new ChatStreamResult(function () use ($response): Generator { $contentSoFar = ''; $toolCallsSoFar = []; + $messageId = null; + $model = null; + $finishReason = null; + $usage = null; + $currentToolCall = null; /** @var \Anthropic\Responses\Messages\CreateStreamedResponse $chunk */ foreach ($response as $chunk) { - // Grab the usage if available - $usage = $chunk->usage !== null - ? $this->mapUsage($chunk->usage) - : null; - - // dump($chunk); - - // There may not be a choice, when for example the usage is returned at the end of the stream. - // if ($chunk->choices !== []) { - // // we only handle a single choice for now when streaming - // $choice = $chunk->choices[0]; - // $contentSoFar .= $choice->delta->content; - - // // Track tool calls across chunks - // foreach ($choice->delta->toolCalls as $toolCall) { - // // If this chunk has an ID and name, it's the start of a new tool call - // if ($toolCall->id !== null && $toolCall->function->name !== null) { - // $toolCallsSoFar[] = [ - // 'id' => $toolCall->id, - // 'function' => [ - // 'name' => $toolCall->function->name, - // 'arguments' => $toolCall->function->arguments, - // ], - // ]; - // } - // // If it has arguments, it belongs to the last tool call - // elseif ($toolCall->function->arguments !== '' && $toolCallsSoFar !== []) { - // $lastIndex = count($toolCallsSoFar) - 1; - // $toolCallsSoFar[$lastIndex]['function']['arguments'] .= $toolCall->function->arguments; - // } - // } - - // $accumulatedToolCallsSoFar = $toolCallsSoFar === [] ? null : new ToolCallCollection( - // collect($toolCallsSoFar) - // ->map(function (array $toolCall): ToolCall { - // try { - // $arguments = (new JsonOutputParser())->parse($toolCall['function']['arguments']); - // } catch (OutputParserException) { - // $arguments = []; - // } - - // return new ToolCall( - // $toolCall['id'], - // new FunctionCall( - // $toolCall['function']['name'], - // $arguments, - // ), - // ); - // }) - // ->values() - // ->all(), - // ); - // } - - // $content = $chunk->delta->type === 'text_delta' ? $chunk->delta->text : null; - - // $finishReason = static::mapFinishReason($choice->finishReason ?? null); - // $isFinal = $finishReason !== null && $usage !== null; - - // $chunk = new ChatGenerationChunk( - // id: $chunk->id, - // message: new AssistantMessage( - // content: $choice->delta->content ?? null, - // toolCalls: $accumulatedToolCallsSoFar ?? null, - // metadata: new ResponseMetadata( - // id: $chunk->id, - // model: $this->model, - // provider: $this->modelProvider, - // finishReason: $finishReason, - // usage: $usage, - // ), - // ), - // index: $choice->index ?? 0, - // createdAt: DateTimeImmutable::createFromFormat('U', (string) $chunk->created), - // finishReason: $finishReason, - // usage: $usage, - // contentSoFar: $contentSoFar, - // isFinal: $isFinal, - // ); - - // $this->dispatchEvent( - // $chunk->isFinal - // ? new ChatModelEnd($chunk) - // : new ChatModelStream($chunk), - // ); + $chunkDelta = null; + $accumulatedToolCallsSoFar = null; + $finishReason = static::mapFinishReason($chunk->delta->stop_reason); + + switch ($chunk->type) { + case 'message_start': + $messageId = $chunk->message->id; + $model = $chunk->message->model; + $usage = $chunk->usage !== null ? $this->mapUsage($chunk->usage) : null; + break; + + case 'content_block_start': + if ($chunk->content_block_start->type === 'tool_use') { + // Start of a new tool call + $currentToolCall = [ + 'id' => $chunk->content_block_start->id, + 'function' => [ + 'name' => $chunk->content_block_start->name, + 'arguments' => '', + ], + ]; + $toolCallsSoFar[] = $currentToolCall; + } + + break; + + case 'content_block_delta': + if ($chunk->delta->type === 'text_delta') { + // Text content delta + $chunkDelta = $chunk->delta->text; + $contentSoFar .= $chunkDelta; + } elseif ($chunk->delta->type === 'input_json_delta' && $currentToolCall !== null) { + // Tool call arguments delta + $lastIndex = count($toolCallsSoFar) - 1; + + if ($lastIndex >= 0) { + $toolCallsSoFar[$lastIndex]['function']['arguments'] .= $chunk->delta->partial_json; + } + } + + break; + + case 'content_block_stop': + // Content block finished - finalize current tool call if applicable + $currentToolCall = null; + break; + + case 'message_delta': + if ($chunk->usage !== null) { + $usage = $this->mapUsage($chunk->usage); + } + + break; + + case 'message_stop': + // Final event - this will be the last chunk + break; + + case 'ping': + // Skip ping events + continue 2; + } + + // Build accumulated tool calls if any exist + if ($toolCallsSoFar !== []) { + $accumulatedToolCallsSoFar = new ToolCallCollection( + collect($toolCallsSoFar) + ->map(function (array $toolCall): ToolCall { + try { + $arguments = json_decode($toolCall['function']['arguments'], true, flags: JSON_THROW_ON_ERROR); + } catch (JsonException) { + $arguments = []; + } + + return new ToolCall( + $toolCall['id'], + new FunctionCall( + $toolCall['function']['name'], + $arguments, + ), + ); + }) + ->values() + ->all(), + ); + } + + $chunk = new ChatGenerationChunk( + id: $messageId ?? 'unknown', + message: new AssistantMessage( + content: $chunkDelta, + toolCalls: $accumulatedToolCallsSoFar, + metadata: new ResponseMetadata( + id: $messageId ?? 'unknown', + model: $model ?? $this->model, + provider: $this->modelProvider, + finishReason: $finishReason, + usage: $usage, + ), + ), + index: 0, + createdAt: new DateTimeImmutable(), + finishReason: $finishReason, + usage: $usage, + contentSoFar: $contentSoFar, + isFinal: $finishReason !== null, + ); + + $chunk = $this->applyOutputParserIfApplicable($chunk); + + $this->dispatchEvent( + $chunk->isFinal + ? new ChatModelEnd($chunk) + : new ChatModelStream($chunk), + ); yield $chunk; } @@ -319,16 +367,16 @@ protected static function mapMessagesForInput(MessageCollection $messages): arra 'type' => 'text', 'text' => $content->text, ], - $content instanceof ImageContent => [ - 'type' => 'image_url', - 'image_url' => [ - 'url' => $content->urlOrBase64, - ], - ], - $content instanceof DocumentContent => [ - 'type' => 'document', - 'document' => $content->data, - ], + // $content instanceof ImageContent => [ + // 'type' => 'image_url', + // 'image_url' => [ + // 'url' => $content->urlOrBase64, + // ], + // ], + // $content instanceof DocumentContent => [ + // 'type' => 'document', + // 'document' => $content->data, + // ], default => $content, }; }, $formattedMessage['content']); @@ -367,56 +415,39 @@ protected function buildParams(array $additionalParameters): array ]; if ($this->structuredOutputConfig !== null) { - $schema = $this->structuredOutputConfig->schema; - $params['response_format'] = [ - 'type' => 'json_schema', - 'json_schema' => [ - 'name' => $this->structuredOutputConfig->name, - 'description' => $this->structuredOutputConfig->description ?? $schema->getDescription(), - 'schema' => $schema instanceof ObjectSchema - ? $schema->additionalProperties(false)->toArray() - : $schema, - 'strict' => $this->structuredOutputConfig->strict, - ], - ]; + $this->structuredOutputMode = StructuredOutputMode::Tool; } elseif ($this->forceJsonOutput) { - $params['response_format'] = [ - 'type' => 'json_object', - ]; + $this->structuredOutputMode = StructuredOutputMode::Json; } if ($this->toolConfig !== null) { - if (is_string($this->toolConfig->toolChoice)) { - $toolChoice = [ - 'type' => 'function', - 'function' => [ - 'name' => $this->toolConfig->toolChoice, - ], - ]; - } else { - $toolChoice = $this->toolConfig->toolChoice->value; - } + $choice = $this->toolConfig->toolChoice; - $params['tool_choice'] = match ($toolChoice) { - ToolChoice::Required->value => 'any', - default => $toolChoice, + $params['tool_choice'] = match (true) { + is_string($choice) => [ + 'type' => 'tool', + 'name' => $choice, + ], + default => [ + 'type' => match ($choice) { + ToolChoice::Required => 'any', + default => $choice, + }, + 'disable_parallel_tool_use' => ! $this->toolConfig->allowParallelToolCalls, + ], }; + // TODO: add ProviderTool support for Anthropic e.g. web_search, etc. $params['tools'] = collect($this->toolConfig->tools) ->map(fn(Tool $tool): array => [ - 'type' => 'function', - 'function' => $tool->format(), + 'type' => 'custom', + 'name' => $tool->name(), + 'description' => $tool->description(), + 'input_schema' => $tool->schema()->additionalProperties(false)->toArray(), ]) ->toArray(); } - // Ensure the usage information is returned when streaming - if ($this->streaming) { - $params['stream_options'] = [ - 'include_usage' => true, - ]; - } - return [ ...$params, ...$this->parameters, diff --git a/src/LLM/Drivers/OpenAIChat.php b/src/LLM/Drivers/OpenAIChat.php index b9025eb..8bae2d7 100644 --- a/src/LLM/Drivers/OpenAIChat.php +++ b/src/LLM/Drivers/OpenAIChat.php @@ -7,7 +7,6 @@ use Generator; use Throwable; use DateTimeImmutable; -use Cortex\Support\Utils; use Cortex\LLM\Data\Usage; use Cortex\LLM\AbstractLLM; use Illuminate\Support\Arr; @@ -22,7 +21,6 @@ use Cortex\LLM\Data\FunctionCall; use Cortex\Events\ChatModelStream; use Cortex\LLM\Enums\FinishReason; -use Cortex\Exceptions\LLMException; use Cortex\LLM\Data\ChatGeneration; use OpenAI\Contracts\ClientContract; use OpenAI\Responses\StreamResponse; @@ -61,15 +59,7 @@ public function invoke( MessageCollection|Message|array|string $messages, array $additionalParameters = [], ): ChatResult|ChatStreamResult { - $messages = Utils::toMessageCollection($messages)->withoutPlaceholders(); - - if ($messages->isEmpty()) { - throw new LLMException('You must provide at least one message to the LLM.'); - } - - if ($this->shouldApplyFormatInstructions && $formatInstructions = $this->outputParser?->formatInstructions()) { - $messages = static::applyFormatInstructions($messages, $formatInstructions); - } + $messages = $this->resolveMessages($messages); $params = $this->buildParams([ ...$additionalParameters, @@ -115,7 +105,9 @@ protected function mapResponse(CreateResponse $response): ChatResult ->map(function (CreateResponseChoice $choice) use ($toolCalls, $finishReason, $usage, $response): ChatGeneration { $generation = new ChatGeneration( message: new AssistantMessage( - content: $choice->message->content, + content: [ + new TextContent($choice->message->content), + ], toolCalls: $toolCalls, metadata: new ResponseMetadata( id: $response->id, @@ -130,16 +122,7 @@ protected function mapResponse(CreateResponse $response): ChatResult finishReason: $finishReason, ); - if ($this->outputParser !== null) { - try { - $parsedOutput = $this->outputParser->parse($generation); - $generation = $generation->cloneWithParsedOutput($parsedOutput); - } catch (OutputParserException $e) { - $this->outputParserError = $e->getMessage(); - } - } - - return $generation; + return $this->applyOutputParserIfApplicable($generation); }) ->all(); @@ -222,7 +205,6 @@ protected function mapStreamResponse(StreamResponse $response): ChatStreamResult } $finishReason = static::mapFinishReason($choice->finishReason ?? null); - $isFinal = $finishReason !== null; // && $usage !== null; $chunk = new ChatGenerationChunk( id: $chunk->id, @@ -242,17 +224,10 @@ protected function mapStreamResponse(StreamResponse $response): ChatStreamResult finishReason: $finishReason, usage: $usage, contentSoFar: $contentSoFar, - isFinal: $isFinal, + isFinal: $finishReason !== null, ); - if ($this->outputParser !== null) { - try { - $parsedOutput = $this->outputParser->parse($chunk); - $chunk = $chunk->cloneWithParsedOutput($parsedOutput); - } catch (OutputParserException $e) { - $this->outputParserError = $e->getMessage(); - } - } + $chunk = $this->applyOutputParserIfApplicable($chunk); $this->dispatchEvent( $chunk->isFinal @@ -422,6 +397,7 @@ protected function buildParams(array $additionalParameters): array } $params['tool_choice'] = $toolChoice; + $params['parallel_tool_calls'] = $this->toolConfig->allowParallelToolCalls; $params['tools'] = collect($this->toolConfig->tools) ->map(fn(Tool $tool): array => [ 'type' => 'function', diff --git a/src/LLM/Drivers/OpenAIResponses.php b/src/LLM/Drivers/OpenAIResponses.php new file mode 100644 index 0000000..67e6bc7 --- /dev/null +++ b/src/LLM/Drivers/OpenAIResponses.php @@ -0,0 +1,532 @@ +resolveMessages($messages); + + $params = $this->buildParams([ + ...$additionalParameters, + 'input' => $this->mapMessagesForInput($messages), + ]); + + $this->dispatchEvent(new ChatModelStart($messages, $params)); + + try { + return $this->streaming + ? $this->mapStreamResponse($this->client->responses()->createStreamed($params)) + : $this->mapResponse($this->client->responses()->create($params)); + } catch (Throwable $e) { + $this->dispatchEvent(new ChatModelError($params, $e)); + + throw $e; + } + } + + /** + * Map a standard (non-streaming) response to a ChatResult. + */ + protected function mapResponse(CreateResponse $response): ChatResult + { + $output = collect($response->output); + $usage = $this->mapUsage($response->usage); + $finishReason = static::mapFinishReason($response->status); + + /** @var \OpenAI\Responses\Responses\Output\OutputMessage $outputMessage */ + $outputMessage = $output->filter(fn(ResponseContract $item): bool => $item instanceof OutputMessage)->first(); + + $outputMessageContent = collect($outputMessage->content); + + if ($outputMessageContent->contains(fn(ResponseContract $item): bool => $item instanceof OutputMessageContentRefusal)) { + throw new LLMException('LLM refusal: ' . $outputMessageContent->first()->refusal); + } + + /** @var \OpenAI\Responses\Responses\Output\OutputMessageContentOutputText $textContent */ + $textContent = $outputMessageContent + ->filter(fn(ResponseContract $item): bool => $item instanceof OutputMessageContentOutputText) + ->first(); + + // TODO: Ignore other provider specific tool calls + // and only support function tool calls for now + $toolCalls = $output + ->filter(fn(ResponseContract $item): bool => $item instanceof OutputFunctionToolCall) + ->map(fn(OutputFunctionToolCall $toolCall): ToolCall => new ToolCall( + $toolCall->id, + new FunctionCall( + $toolCall->name, + json_decode($toolCall->arguments, true, flags: JSON_THROW_ON_ERROR), + ), + )) + ->all(); + + $reasoningContent = $output + ->filter(fn(ResponseContract $item): bool => $item instanceof OutputReasoning) + ->map(fn(OutputReasoning $reasoning): ReasoningContent => new ReasoningContent( + $reasoning->id, + Arr::first($reasoning->summary)->text ?? '', + )) + ->all(); + + $generation = new ChatGeneration( + message: new AssistantMessage( + content: [ + ...$reasoningContent, + new TextContent($textContent->text), + ], + toolCalls: new ToolCallCollection($toolCalls), + metadata: new ResponseMetadata( + id: $response->id, + model: $response->model, + provider: $this->modelProvider, + finishReason: $finishReason, + usage: $usage, + ), + id: $outputMessage->id, + ), + index: 0, + createdAt: DateTimeImmutable::createFromFormat('U', (string) $response->createdAt), + finishReason: $finishReason, + ); + + $generation = $this->applyOutputParserIfApplicable($generation); + + /** @var array $rawResponse */ + $rawResponse = $response->toArray(); + + $result = new ChatResult( + [$generation], + $usage, + $rawResponse, // @phpstan-ignore argument.type + ); + + $this->dispatchEvent(new ChatModelEnd($result)); + + return $result; + } + + /** + * Map a streaming response to a ChatStreamResult. + * + * @param StreamResponse<\OpenAI\Responses\Responses\CreateStreamedResponse> $response + * + * @return \Cortex\LLM\Data\ChatStreamResult<\Cortex\LLM\Data\ChatGenerationChunk> + */ + protected function mapStreamResponse(StreamResponse $response): ChatStreamResult + { + throw new Exception('Not implemented'); + // return new ChatStreamResult(function () use ($response): Generator { + // $contentSoFar = ''; + // $toolCallsSoFar = []; + + // /** @var \OpenAI\Responses\Responses\CreateStreamedResponse $chunk */ + // foreach ($response as $chunk) { + // // Grab the usage if available + // $usage = $chunk->usage !== null + // ? $this->mapUsage($chunk->usage) + // : null; + + // // There may not be a choice, when for example the usage is returned at the end of the stream. + // if ($chunk->choices !== []) { + // // we only handle a single choice for now when streaming + // $choice = $chunk->choices[0]; + // $contentSoFar .= $choice->delta->content; + + // // Track tool calls across chunks + // foreach ($choice->delta->toolCalls as $toolCall) { + // // If this chunk has an ID and name, it's the start of a new tool call + // if ($toolCall->id !== null && $toolCall->function->name !== null) { + // $toolCallsSoFar[] = [ + // 'id' => $toolCall->id, + // 'function' => [ + // 'name' => $toolCall->function->name, + // 'arguments' => $toolCall->function->arguments, + // ], + // ]; + // } + // // If it has arguments, it belongs to the last tool call + // elseif ($toolCall->function->arguments !== '' && $toolCallsSoFar !== []) { + // $lastIndex = count($toolCallsSoFar) - 1; + // $toolCallsSoFar[$lastIndex]['function']['arguments'] .= $toolCall->function->arguments; + // } + // } + + // $accumulatedToolCallsSoFar = $toolCallsSoFar === [] ? null : new ToolCallCollection( + // collect($toolCallsSoFar) + // ->map(function (array $toolCall): ToolCall { + // try { + // $arguments = (new JsonOutputParser())->parse($toolCall['function']['arguments']); + // } catch (OutputParserException) { + // $arguments = []; + // } + + // return new ToolCall( + // $toolCall['id'], + // new FunctionCall( + // $toolCall['function']['name'], + // $arguments, + // ), + // ); + // }) + // ->values() + // ->all(), + // ); + // } + + // $finishReason = static::mapFinishReason($choice->finishReason ?? null); + + // $chunk = new ChatGenerationChunk( + // id: $chunk->id, + // message: new AssistantMessage( + // content: $choice->delta->content ?? null, + // toolCalls: $accumulatedToolCallsSoFar ?? null, + // metadata: new ResponseMetadata( + // id: $chunk->id, + // model: $this->model, + // provider: $this->modelProvider, + // finishReason: $finishReason, + // usage: $usage, + // ), + // ), + // index: $choice->index ?? 0, + // createdAt: DateTimeImmutable::createFromFormat('U', (string) $chunk->created), + // finishReason: $finishReason, + // usage: $usage, + // contentSoFar: $contentSoFar, + // isFinal: $finishReason !== null, + // ); + + // $chunk = $this->applyOutputParserIfApplicable($chunk); + + // $this->dispatchEvent( + // $chunk->isFinal + // ? new ChatModelEnd($chunk) + // : new ChatModelStream($chunk), + // ); + + // yield $chunk; + // } + // }); + } + + /** + * Map the OpenAI usage response to a Usage object. + */ + protected function mapUsage(CreateResponseUsage $usage): Usage + { + return new Usage( + promptTokens: $usage->inputTokens, + completionTokens: $usage->outputTokens, + cachedTokens: $usage->inputTokensDetails->cachedTokens, + reasoningTokens: $usage->outputTokensDetails->reasoningTokens, + totalTokens: $usage->totalTokens, + inputCost: $this->modelProvider->inputCostForTokens($this->model, $usage->inputTokens), + outputCost: $this->modelProvider->outputCostForTokens($this->model, $usage->outputTokens), + ); + } + + /** + * Take the given messages and format them for the OpenAI Responses API. + * + * @return array> + */ + protected function mapMessagesForInput(MessageCollection $messages): array + { + return $messages + ->map(function (Message $message): array { + // Handle ToolMessage specifically for Responses API + if ($message instanceof ToolMessage) { + return [ + 'role' => $message->role->value, + 'content' => $this->mapContentForResponsesAPI($message->content()), + 'tool_call_id' => $message->id, + ]; + } + + // Handle AssistantMessage with tool calls + if ($message instanceof AssistantMessage && $message->toolCalls?->isNotEmpty()) { + $baseMessage = [ + 'role' => $message->role->value, + ]; + + // Add content if present + if ($message->content() !== null) { + $baseMessage['content'] = $this->mapContentForResponsesAPI($message->content()); + } + + // Add tool calls + $baseMessage['tool_calls'] = $message->toolCalls->map(function (ToolCall $toolCall): array { + return [ + 'id' => $toolCall->id, + 'type' => 'function', + 'function' => [ + 'name' => $toolCall->function->name, + 'arguments' => json_encode($toolCall->function->arguments), + ], + ]; + })->toArray(); + + return $baseMessage; + } + + // Handle all other message types + $formattedMessage = [ + 'role' => $message->role()->value, + 'content' => $this->mapContentForResponsesAPI($message->content()), + ]; + + return $formattedMessage; + }) + ->values() + ->toArray(); + } + + /** + * Map content to the OpenAI Responses API format. + * + * @param string|array|null $content + * + * @return array> + */ + protected function mapContentForResponsesAPI(string|array|null $content): array + { + if ($content === null) { + return []; + } + + if (is_string($content)) { + return [ + [ + 'type' => 'input_text', + 'text' => $content, + ], + ]; + } + + return array_map(function (mixed $item): array { + if ($item instanceof ImageContent) { + $this->supportsFeatureOrFail(ModelFeature::Vision); + + return [ + 'type' => 'input_image', + 'image_url' => $item->url, + 'detail' => 'auto', // Default detail level + ]; + } + + if ($item instanceof AudioContent) { + $this->supportsFeatureOrFail(ModelFeature::AudioInput); + + return [ + 'type' => 'input_audio', + 'data' => $item->base64Data, + 'format' => $item->format, + ]; + } + + if ($item instanceof FileContent) { + return [ + 'type' => 'input_file', + 'file_id' => $item->fileName, // Assuming file_id should be the fileName + ]; + } + + if ($item instanceof TextContent) { + return [ + 'type' => 'input_text', + 'text' => $item->text ?? '', + ]; + } + + // Handle ReasoningContent and ToolContent + if ($item instanceof ReasoningContent) { + return [ + 'type' => 'input_text', + 'text' => $item->reasoning, + ]; + } + + // Fallback for unknown content types + return []; + }, $content); + } + + protected static function mapFinishReason(?string $finishReason): ?FinishReason + { + if ($finishReason === null) { + return null; + } + + return match ($finishReason) { + 'stop' => FinishReason::Stop, + 'length' => FinishReason::Length, + 'content_filter' => FinishReason::ContentFilter, + 'tool_calls' => FinishReason::ToolCalls, + default => FinishReason::Unknown, + }; + } + + /** + * @param array $additionalParameters + * + * @return array + */ + protected function buildParams(array $additionalParameters): array + { + $params = [ + 'model' => $this->model, + ]; + + if ($this->structuredOutputConfig !== null) { + $this->supportsFeatureOrFail(ModelFeature::StructuredOutput); + + $schema = $this->structuredOutputConfig->schema; + $params['response_format'] = [ + 'type' => 'json_schema', + 'json_schema' => [ + 'name' => $this->structuredOutputConfig->name, + 'description' => $this->structuredOutputConfig->description ?? $schema->getDescription(), + 'schema' => $schema->additionalProperties(false)->toArray(), + 'strict' => $this->structuredOutputConfig->strict, + ], + ]; + } elseif ($this->forceJsonOutput) { + $params['response_format'] = [ + 'type' => 'json_object', + ]; + } + + if ($this->toolConfig !== null) { + $this->supportsFeatureOrFail(ModelFeature::ToolCalling); + + if (is_string($this->toolConfig->toolChoice)) { + $toolChoice = [ + 'type' => 'function', + 'function' => [ + 'name' => $this->toolConfig->toolChoice, + ], + ]; + } else { + $toolChoice = $this->toolConfig->toolChoice->value; + } + + $params['tool_choice'] = $toolChoice; + $params['parallel_tool_calls'] = $this->toolConfig->allowParallelToolCalls; + $params['tools'] = collect($this->toolConfig->tools) + ->map(fn(Tool $tool): array => [ + 'type' => 'function', + 'function' => $tool->format(), + ]) + ->toArray(); + } + + // Ensure the usage information is returned when streaming + if ($this->streaming) { + $params['stream_options'] = [ + 'include_usage' => true, + ]; + } + + $allParams = [ + ...$params, + ...$this->parameters, + ...$additionalParameters, + ]; + + if ($this->modelProvider === ModelProvider::OpenAI) { + if (array_key_exists('max_tokens', $allParams)) { + // Backwards compatibility for `max_tokens` + $allParams['max_output_tokens'] = $allParams['max_tokens']; + unset($allParams['max_tokens']); + } + + if (array_key_exists('temperature', $allParams) && $allParams['temperature'] === null) { + unset($allParams['temperature']); + } + + if (array_key_exists('top_p', $allParams) && $allParams['top_p'] === null) { + unset($allParams['top_p']); + } + } + + return $allParams; + } + + public function getClient(): ClientContract + { + return $this->client; + } + + /** + * @param array|\OpenAI\Responses\StreamResponse<\OpenAI\Responses\Chat\CreateStreamedResponse>|string> $responses + * + * @phpstan-ignore-next-line + */ + public static function fake(array $responses, ?string $model = null, ?ModelProvider $modelProvider = null): self + { + $client = new ClientFake($responses); + + return new self( + $client, + $model ?? CreateResponseFixture::ATTRIBUTES['model'], + $modelProvider ?? ModelProvider::OpenAI, + ); + } +} diff --git a/src/LLM/LLMManager.php b/src/LLM/LLMManager.php index eda613f..f3fe221 100644 --- a/src/LLM/LLMManager.php +++ b/src/LLM/LLMManager.php @@ -13,7 +13,9 @@ use InvalidArgumentException; use Illuminate\Support\Manager; use Cortex\LLM\Drivers\OpenAIChat; +use OpenAI\Contracts\ClientContract; use Cortex\LLM\Drivers\AnthropicChat; +use Cortex\LLM\Drivers\OpenAIResponses; use Cortex\ModelInfo\Enums\ModelProvider; use Cortex\Support\IlluminateEventDispatcherBridge; @@ -60,36 +62,33 @@ protected function createDriver($driver): LLM // @pest-ignore-type } /** - * Create an OpenAI LLM driver instance. + * Create an OpenAI Chat LLM driver instance. * * @param array{default_model?: string, model_provider?: string, default_parameters: array, options: array{api_key?: string, organization?: string, base_uri?: string, headers?: array, query_params?: array}} $config */ public function createOpenAIDriver(array $config, string $name): OpenAIChat|CacheDecorator { - $client = OpenAI::factory()->withApiKey(Arr::get($config, 'options.api_key', '')); - - if ($organization = Arr::get($config, 'options.organization')) { - $client->withOrganization($organization); - } - - if ($baseUri = Arr::get($config, 'options.base_uri')) { - $client->withBaseUri($baseUri); - } - - foreach (Arr::get($config, 'options.headers', []) as $key => $value) { - $client->withHttpHeader($key, $value); - } + $driver = new OpenAIChat( + $this->buildOpenAIClient($config), + $config['default_model'], + $this->getModelProviderFromConfig($config, $name), + ); - foreach (Arr::get($config, 'options.query_params', []) as $key => $value) { - $client->withQueryParam($key, $value); - } + $driver->withParameters(Arr::get($config, 'default_parameters', [])); + $driver->setEventDispatcher(new IlluminateEventDispatcherBridge($this->container->make('events'))); - if (! isset($config['default_model'])) { - throw new InvalidArgumentException('default_model is required.'); - } + return $this->getCacheDecorator($driver) ?? $driver; + } - $driver = new OpenAIChat( - $client->make(), + /** + * Create an OpenAI Responses LLM driver instance. + * + * @param array{default_model?: string, model_provider?: string, default_parameters: array, options: array{api_key?: string, organization?: string, base_uri?: string, headers?: array, query_params?: array}} $config + */ + public function createOpenAIResponsesDriver(array $config, string $name): OpenAIResponses + { + $driver = new OpenAIResponses( + $this->buildOpenAIClient($config), $config['default_model'], $this->getModelProviderFromConfig($config, $name), ); @@ -103,7 +102,7 @@ public function createOpenAIDriver(array $config, string $name): OpenAIChat|Cach /** * Create an Anthropic LLM driver instance. * - * @param array $config + * @param array{default_model?: string, model_provider?: string, default_parameters: array, options: array{api_key?: string, organization?: string, base_uri?: string, headers?: array, query_params?: array}} $config */ public function createAnthropicDriver(array $config, string $name): AnthropicChat|CacheDecorator { @@ -141,6 +140,32 @@ public function createAnthropicDriver(array $config, string $name): AnthropicCha return $this->getCacheDecorator($driver) ?? $driver; } + /** + * @param array{options: array{api_key?: string, organization?: string, base_uri?: string, headers?: array, query_params?: array}} $config + */ + protected function buildOpenAIClient(array $config): ClientContract + { + $client = OpenAI::factory()->withApiKey(Arr::get($config, 'options.api_key', '')); + + if ($organization = Arr::get($config, 'options.organization')) { + $client->withOrganization($organization); + } + + if ($baseUri = Arr::get($config, 'options.base_uri')) { + $client->withBaseUri($baseUri); + } + + foreach (Arr::get($config, 'options.headers', []) as $key => $value) { + $client->withHttpHeader($key, $value); + } + + foreach (Arr::get($config, 'options.query_params', []) as $key => $value) { + $client->withQueryParam($key, $value); + } + + return $client->make(); + } + /** * @param array{model_provider?: string} $config * diff --git a/src/Prompts/Builders/Concerns/BuildsPrompts.php b/src/Prompts/Builders/Concerns/BuildsPrompts.php index d7e2498..b62b0dd 100644 --- a/src/Prompts/Builders/Concerns/BuildsPrompts.php +++ b/src/Prompts/Builders/Concerns/BuildsPrompts.php @@ -14,6 +14,7 @@ use Cortex\JsonSchema\Types\ObjectSchema; use Cortex\LLM\Data\StructuredOutputConfig; use Cortex\Prompts\Contracts\PromptTemplate; +use Cortex\Tasks\Enums\StructuredOutputMode; trait BuildsPrompts { @@ -49,6 +50,7 @@ public function metadata( array $parameters = [], array $tools = [], StructuredOutputConfig|ObjectSchema|string|null $structuredOutput = null, + StructuredOutputMode $structuredOutputMode = StructuredOutputMode::Auto, array $additional = [], ): self { return $this->setMetadata(new PromptMetadata( @@ -57,6 +59,7 @@ public function metadata( $parameters, $tools, $structuredOutput, + $structuredOutputMode, $additional, )); } @@ -119,7 +122,7 @@ public function llm( } /** - * Convenience method to build and pipe the prompt template. + * Convenience method to build and pipe the prompt template to a given pipeable. */ public function pipe(Pipeable|callable $pipeable): Pipeline { diff --git a/src/Prompts/Contracts/PromptBuilder.php b/src/Prompts/Contracts/PromptBuilder.php index 2698dc2..e47a1fd 100644 --- a/src/Prompts/Contracts/PromptBuilder.php +++ b/src/Prompts/Contracts/PromptBuilder.php @@ -9,6 +9,7 @@ use Cortex\LLM\Contracts\LLM; use Cortex\JsonSchema\Types\ObjectSchema; use Cortex\LLM\Data\StructuredOutputConfig; +use Cortex\Tasks\Enums\StructuredOutputMode; interface PromptBuilder { @@ -33,6 +34,7 @@ public function metadata( array $parameters = [], array $tools = [], StructuredOutputConfig|ObjectSchema|string|null $structuredOutput = null, + StructuredOutputMode $structuredOutputMode = StructuredOutputMode::Auto, array $additional = [], ): self; } diff --git a/src/Prompts/Contracts/PromptTemplate.php b/src/Prompts/Contracts/PromptTemplate.php index 27b404d..e26cd18 100644 --- a/src/Prompts/Contracts/PromptTemplate.php +++ b/src/Prompts/Contracts/PromptTemplate.php @@ -7,6 +7,7 @@ use Closure; use Cortex\Pipeline; use Cortex\LLM\Contracts\LLM; +use Cortex\Contracts\Pipeable; use Illuminate\Support\Collection; interface PromptTemplate @@ -26,7 +27,12 @@ public function format(?array $variables = null): mixed; public function variables(): Collection; /** - * Convenience method to pipe the prompt template to an LLM. + * Convenience method to build and pipe the prompt template to an LLM. */ public function llm(LLM|string|null $provider = null, Closure|string|null $model = null): Pipeline; + + /** + * Convenience method to build and pipe the prompt template to a given pipeable. + */ + public function pipe(Pipeable|callable $pipeable): Pipeline; } diff --git a/src/Prompts/Data/PromptMetadata.php b/src/Prompts/Data/PromptMetadata.php index 1a9a85e..1a9aaf9 100644 --- a/src/Prompts/Data/PromptMetadata.php +++ b/src/Prompts/Data/PromptMetadata.php @@ -6,6 +6,7 @@ use Cortex\JsonSchema\Types\ObjectSchema; use Cortex\LLM\Data\StructuredOutputConfig; +use Cortex\Tasks\Enums\StructuredOutputMode; readonly class PromptMetadata { @@ -20,6 +21,7 @@ public function __construct( public array $parameters = [], public array $tools = [], public StructuredOutputConfig|ObjectSchema|string|null $structuredOutput = null, + public StructuredOutputMode $structuredOutputMode = StructuredOutputMode::Auto, public array $additional = [], ) {} } diff --git a/src/Prompts/Factories/LangfusePromptFactory.php b/src/Prompts/Factories/LangfusePromptFactory.php index d9b0173..a343927 100644 --- a/src/Prompts/Factories/LangfusePromptFactory.php +++ b/src/Prompts/Factories/LangfusePromptFactory.php @@ -17,6 +17,7 @@ use Cortex\LLM\Data\Messages\SystemMessage; use Cortex\Prompts\Contracts\PromptFactory; use Cortex\Prompts\Contracts\PromptTemplate; +use Cortex\Tasks\Enums\StructuredOutputMode; use Cortex\LLM\Data\Messages\AssistantMessage; use Cortex\Prompts\Builders\ChatPromptBuilder; use Cortex\Prompts\Builders\TextPromptBuilder; @@ -206,18 +207,21 @@ protected function getResponseContent(string $name, array $options = []): array protected static function defaultMetadataResolver(): Closure { return function (array $config): PromptMetadata { - $structuredOutput = $config['structuredOutput'] ?? null; + $structuredOutput = $config['structured_output'] ?? null; if (is_string($structuredOutput) || is_array($structuredOutput)) { $structuredOutput = SchemaFactory::fromJson($structuredOutput); } + $structuredOutputMode = StructuredOutputMode::tryFrom($config['structured_output_mode'] ?? ''); + return new PromptMetadata( $config['provider'] ?? null, $config['model'] ?? null, $config['parameters'] ?? [], $config['tools'] ?? [], $structuredOutput, + $structuredOutputMode ?? StructuredOutputMode::Auto, $config['additional'] ?? [], ); }; diff --git a/src/Prompts/Templates/AbstractPromptTemplate.php b/src/Prompts/Templates/AbstractPromptTemplate.php index 4670e3c..63f8186 100644 --- a/src/Prompts/Templates/AbstractPromptTemplate.php +++ b/src/Prompts/Templates/AbstractPromptTemplate.php @@ -108,9 +108,13 @@ public function llm( $this->metadata->structuredOutput->name, $this->metadata->structuredOutput->description, $this->metadata->structuredOutput->strict, + $this->metadata->structuredOutputMode, ); } else { - $llm->withStructuredOutput($this->metadata->structuredOutput); + $llm->withStructuredOutput( + output: $this->metadata->structuredOutput, + outputMode: $this->metadata->structuredOutputMode, + ); } } diff --git a/src/Support/Utils.php b/src/Support/Utils.php index 24b91ad..4b0a14a 100644 --- a/src/Support/Utils.php +++ b/src/Support/Utils.php @@ -12,7 +12,7 @@ use Illuminate\Support\Collection; use Cortex\Exceptions\ContentException; use Cortex\Exceptions\GenericException; -use Cortex\JsonSchema\Contracts\Schema; +use Cortex\JsonSchema\Types\ObjectSchema; use Cortex\LLM\Data\Messages\UserMessage; use Cortex\LLM\Data\Messages\MessageCollection; use Cortex\LLM\Data\Messages\MessagePlaceholder; @@ -47,7 +47,7 @@ public static function findVariables(string $input): array /** * Resolve tool instances from the given array. * - * @param array $tools + * @param array $tools * * @return \Illuminate\Support\Collection */ @@ -58,7 +58,7 @@ public static function toToolCollection(array $tools): Collection ->map(fn(mixed $tool): object => match (true) { $tool instanceof Tool => $tool, $tool instanceof Closure => new ClosureTool($tool), - $tool instanceof Schema => new SchemaTool($tool), + $tool instanceof ObjectSchema => new SchemaTool($tool), is_string($tool) && class_exists($tool) => resolve($tool), // @phpstan-ignore function.alreadyNarrowedType default => throw new GenericException('Invalid tool'), }); diff --git a/src/Support/helpers.php b/src/Support/helpers.php index 29608db..81186ec 100644 --- a/src/Support/helpers.php +++ b/src/Support/helpers.php @@ -6,20 +6,21 @@ use Closure; use Cortex\Cortex; +use Cortex\Prompts\Prompt; use Cortex\LLM\Contracts\LLM; use Cortex\Tools\ClosureTool; use Cortex\Tasks\Enums\TaskType; use Cortex\Tasks\Builders\TextTaskBuilder; -use Cortex\Prompts\Builders\ChatPromptBuilder; +use Cortex\Prompts\Contracts\PromptBuilder; use Cortex\LLM\Data\Messages\MessageCollection; use Cortex\Tasks\Builders\StructuredTaskBuilder; /** * Helper function to create a chat prompt builder. * - * @param MessageCollection|array|string $messages + * @param MessageCollection|array|string|null $messages */ -function prompt(MessageCollection|array|string $messages): ChatPromptBuilder +function prompt(MessageCollection|array|string|null $messages): Prompt|PromptBuilder { return Cortex::prompt($messages); } diff --git a/src/Tasks/AbstractTask.php b/src/Tasks/AbstractTask.php index 22d3b57..aa24b7a 100644 --- a/src/Tasks/AbstractTask.php +++ b/src/Tasks/AbstractTask.php @@ -95,11 +95,20 @@ public function pipeline(): Pipeline $tools = $this->getTools(); $outputParser = $this->outputParser(); - return $this->executionPipeline() + // If the output parser is set, we don't want to parse the output + // as part of the ChatResult, as it will be parsed as part of the next pipeable. + $shouldParseOutput = $outputParser === null; + + return $this->executionPipeline($shouldParseOutput) ->when( $tools->isNotEmpty(), fn(Pipeline $pipeline): Pipeline => $pipeline->pipe( - new HandleToolCalls($tools, $this->memory, $this->executionPipeline(), $this->maxIterations), + new HandleToolCalls( + $tools, + $this->memory, + $this->executionPipeline($shouldParseOutput), + $this->maxIterations, + ), ), ) ->when( @@ -111,10 +120,16 @@ public function pipeline(): Pipeline /** * This is the main pipeline that will be used to generate the output. */ - public function executionPipeline(): Pipeline + public function executionPipeline(bool $shouldParseOutput = true): Pipeline { + $llm = $this->llm(); + + if ($shouldParseOutput === false) { + $llm = $llm->shouldParseOutput(false); + } + return $this->prompt() - ->pipe($this->llm()) + ->pipe($llm) ->pipe(new AddMessageToMemory($this->memory)) ->pipe(new AppendUsage($this->usage)); } diff --git a/src/Tools/ClosureTool.php b/src/Tools/ClosureTool.php index 9f134fe..d0e97d7 100644 --- a/src/Tools/ClosureTool.php +++ b/src/Tools/ClosureTool.php @@ -9,14 +9,14 @@ use Cortex\Attributes\Tool; use Cortex\LLM\Data\ToolCall; use Cortex\JsonSchema\SchemaFactory; -use Cortex\JsonSchema\Contracts\Schema; use Cortex\JsonSchema\Support\DocParser; +use Cortex\JsonSchema\Types\ObjectSchema; class ClosureTool extends AbstractTool { protected ReflectionFunction $reflection; - protected Schema $schema; + protected ObjectSchema $schema; public function __construct( protected Closure $closure, @@ -37,7 +37,7 @@ public function description(): string return $this->description ?? $this->resolveDescription(); } - public function schema(): Schema + public function schema(): ObjectSchema { return $this->schema; } diff --git a/src/Tools/SchemaTool.php b/src/Tools/SchemaTool.php index 1e2e6e0..277f3e1 100644 --- a/src/Tools/SchemaTool.php +++ b/src/Tools/SchemaTool.php @@ -6,12 +6,12 @@ use Cortex\LLM\Data\ToolCall; use Cortex\Exceptions\GenericException; -use Cortex\JsonSchema\Contracts\Schema; +use Cortex\JsonSchema\Types\ObjectSchema; class SchemaTool extends AbstractTool { public function __construct( - protected Schema $schema, + protected ObjectSchema $schema, protected ?string $name = null, protected ?string $description = null, ) {} @@ -30,7 +30,7 @@ public function description(): string ?? ''; } - public function schema(): Schema + public function schema(): ObjectSchema { return $this->schema; } diff --git a/tests/Unit/Experimental/PlaygroundTest.php b/tests/Unit/Experimental/PlaygroundTest.php index 5557f7d..f2623cc 100644 --- a/tests/Unit/Experimental/PlaygroundTest.php +++ b/tests/Unit/Experimental/PlaygroundTest.php @@ -350,7 +350,7 @@ function (int $a, int $b): int { test('reasoning output', function (): void { $pipeline = prompt('What is the weight of the moon?') - ->pipe(llm('ollama', 'deepseek-r1:32b')) + ->llm('ollama', 'deepseek-r1:32b') ->pipe(new XmlTagOutputParser('think')); $result = $pipeline->stream(); @@ -421,11 +421,62 @@ enum Sentiment: string })->skip(); test('anthropic real', function (): void { - $result = prompt('Tell me a joke about a chicken') - ->llm('anthropic') - ->invoke(); + Event::listen(ChatModelStart::class, function (ChatModelStart $event): void { + dump($event->parameters); + }); - dd($result); + $prompt = Cortex::prompt() + ->builder() + ->messages([ + new SystemMessage('You are an expert story ideator.'), + new UserMessage('Generate a story idea about {topic}. Only answer in a single sentence.'), + // new UserMessage('What is the weight of the moon? Think step by step.'), + ]) + ->metadata( + provider: 'anthropic', + model: 'claude-sonnet-4-20250514', + structuredOutput: SchemaFactory::object()->properties( + SchemaFactory::string('story_idea')->required(), + ), + structuredOutputMode: StructuredOutputMode::Json, + // parameters: [ + // 'max_tokens' => 3000, + // 'thinking' => [ + // 'type' => 'enabled', + // 'budget_tokens' => 2000, + // ], + // ], + ); + + // dd($prompt->llm()->invoke([ + // 'topic' => 'dragons', + // ])); + + $result = $prompt->llm()->stream([ + 'topic' => 'trolls', + ]); + + foreach ($result as $chunk) { + dump($chunk->parsedOutput); + } + + dd('done'); + + $tellAJoke = $prompt->llm( + 'anthropic', + fn(LLMContract $llm): \Cortex\LLM\Contracts\LLM => $llm->withStructuredOutput( + output: SchemaFactory::object() + ->properties( + SchemaFactory::string('setup')->required(), + SchemaFactory::string('punchline')->required(), + ), + outputMode: StructuredOutputMode::Json, + ), + ); + + dd($tellAJoke->invoke([ + 'topic' => 'dragons', + ])->parsedOutput); })->skip(); test('model info', function (): void { @@ -441,3 +492,20 @@ enum Sentiment: string dd($result); })->skip(); + +test('openai responses', function (): void { + Event::listen(ChatModelStart::class, function (ChatModelStart $event): void { + dump($event->parameters); + }); + + $result = LLM::provider('openai') + ->withModel('gpt-5-mini') + ->withParameters([ + 'reasoning' => [ + 'effort' => 'low', + ], + ]) + ->invoke('How much wood would a woodchuck chuck?'); + + dd($result); +})->skip(); diff --git a/tests/Unit/LLM/Drivers/AnthropicChatTest.php b/tests/Unit/LLM/Drivers/AnthropicChatTest.php index c0d675e..bda75c9 100644 --- a/tests/Unit/LLM/Drivers/AnthropicChatTest.php +++ b/tests/Unit/LLM/Drivers/AnthropicChatTest.php @@ -4,11 +4,23 @@ namespace Cortex\Tests\Unit\LLM\Drivers; +use Cortex\Cortex; +use Cortex\LLM\Data\Usage; +use Cortex\Attributes\Tool; +use Cortex\LLM\Data\ToolCall; use Cortex\LLM\Data\ChatResult; +use Cortex\LLM\Data\FunctionCall; +use Cortex\JsonSchema\SchemaFactory; use Cortex\LLM\Data\ChatStreamResult; use Cortex\LLM\Drivers\AnthropicChat; +use Cortex\LLM\Data\ToolCallCollection; +use Cortex\LLM\Data\ChatGenerationChunk; +use Cortex\ModelInfo\Enums\ModelFeature; use Cortex\LLM\Data\Messages\UserMessage; +use Cortex\Tasks\Enums\StructuredOutputMode; +use Anthropic\Responses\Meta\MetaInformation; use Cortex\LLM\Data\Messages\AssistantMessage; +use Cortex\LLM\Data\Messages\Content\TextContent; use Anthropic\Responses\Messages\CreateResponse as ChatCreateResponse; use Anthropic\Responses\Messages\CreateStreamedResponse as ChatCreateStreamedResponse; @@ -29,10 +41,17 @@ ]); expect($result)->toBeInstanceOf(ChatResult::class) - ->and($result->rawResponse)->toBeArray()->not->toBeEmpty() - ->and($result->generations)->toHaveCount(1) - ->and($result->generation->message)->toBeInstanceOf(AssistantMessage::class) - ->and($result->generation->message->content)->toBe('I am doing well, thank you for asking!'); + ->and($result->rawResponse) + ->toBeArray()->not->toBeEmpty() + ->and($result->generations) + ->toHaveCount(1) + ->and($result->generation->message) + ->toBeInstanceOf(AssistantMessage::class) + ->and($result->generation->message->content) + ->toBeArray() + ->toContainOnlyInstancesOf(TextContent::class) + ->and($result->generation->message->content[0]->text) + ->toBe('I am doing well, thank you for asking!'); }); test('it can stream', function (): void { @@ -48,12 +67,436 @@ expect($chunks)->toBeInstanceOf(ChatStreamResult::class); - // foreach ($chunks as $chunk) { - // dump($chunk); - // } + $output = $chunks->reduce(function (string $carry, ChatGenerationChunk $chunk) { + expect($chunk)->toBeInstanceOf(ChatGenerationChunk::class) + ->and($chunk->message)->toBeInstanceOf(AssistantMessage::class); - // expect($chunks)->toBeInstanceOf(ChatStreamResult::class) - // ->and($chunks->generations)->toHaveCount(1) - // ->and($chunks->generation->message)->toBeInstanceOf(AssistantMessage::class) - // ->and($chunks->generation->message->content)->toBe('Hello!'); + return $carry . ($chunk->message->content ?? ''); + }, ''); + + expect($output)->toBe('Hello!'); +}); + +test('it can use tools', function (): void { + $llm = AnthropicChat::fake([ + createAnthropicResponse([ + 'content' => [ + [ + 'type' => 'tool_use', + 'id' => 'call_123', + 'name' => 'multiply', + 'input' => [ + 'x' => 3, + 'y' => 4, + ], + ], + ], + ]), + ]); + + $llm->addFeature(ModelFeature::ToolCalling); + + $llm->withTools([ + #[Tool(name: 'multiply', description: 'Multiply two numbers')] + fn(int $x, int $y): int => $x * $y, + ]); + + $result = $llm->invoke([ + new UserMessage('What is 3 times 4?'), + ]); + + expect($result->generation->message->toolCalls) + ->toBeInstanceOf(ToolCallCollection::class) + ->and($result->generation->message->toolCalls) + ->toHaveCount(1) + ->and($result->generation->message->toolCalls[0]) + ->toBeInstanceOf(ToolCall::class) + ->and($result->generation->message->toolCalls[0]->function) + ->toBeInstanceOf(FunctionCall::class) + ->and($result->generation->message->toolCalls[0]->function->name) + ->toBe('multiply') + ->and($result->generation->message->toolCalls[0]->function->arguments) + ->toBe([ + 'x' => 3, + 'y' => 4, + ]); +}); + +test('it can use structured output', function (): void { + $response = createAnthropicResponse([ + 'stop_reason' => 'end_turn', + 'content' => [ + [ + 'type' => 'text', + 'text' => '{"name":"John Doe","age":30}', + ], + ], + ]); + + $llm = AnthropicChat::fake([ + $response, + ], 'claude-3-5-sonnet-20241022'); + + $llm->addFeature(ModelFeature::StructuredOutput); + + $llm->withStructuredOutput( + SchemaFactory::object()->properties( + SchemaFactory::string('name'), + SchemaFactory::integer('age'), + ), + name: 'Person', + description: 'A person with a name and age', + ); + + $result = $llm->invoke([ + new UserMessage('Tell me about a person'), + ]); + + expect($result->generation->message->content) + ->toBeArray() + ->toContainOnlyInstancesOf(TextContent::class) + ->and($result->generation->message->content[0]->text) + ->toBe('{"name":"John Doe","age":30}'); + + expect($result->generation->parsedOutput)->toBe([ + 'name' => 'John Doe', + 'age' => 30, + ]); }); + +test('it can use structured output using the schema tool', function (): void { + $response = createAnthropicResponse([ + 'stop_reason' => 'tool_use', + 'content' => [ + [ + 'type' => 'tool_use', + 'id' => 'call_123', + 'name' => 'Person', + 'input' => [ + 'name' => 'John Doe', + 'age' => 30, + ], + ], + ], + ]); + + $llm = AnthropicChat::fake([ + $response, + ], 'claude-3-5-sonnet-20241022'); + + $llm->addFeature(ModelFeature::ToolCalling); + + $schema = SchemaFactory::object('Person')->properties( + SchemaFactory::string('name'), + SchemaFactory::integer('age'), + ); + + $llm->withStructuredOutput($schema, outputMode: StructuredOutputMode::Tool); + + $result = $llm->invoke([ + new UserMessage('Tell me about a person'), + ]); + + expect($result->parsedOutput) + ->toBe([ + 'name' => 'John Doe', + 'age' => 30, + ]); + + /** @var \Anthropic\Testing\ClientFake $client */ + $client = $llm->getClient(); + + $client->messages()->assertSent(function (string $method, array $parameters): bool { + return $parameters['model'] === 'claude-3-5-sonnet-20241022' + && $parameters['messages'][0]['role'] === 'user' + && $parameters['messages'][0]['content'] === 'Tell me about a person' + && $parameters['tools'][0]['type'] === 'custom' + && $parameters['tools'][0]['name'] === 'Person' + && $parameters['tools'][0]['input_schema']['properties']['name']['type'] === 'string' + && $parameters['tools'][0]['input_schema']['properties']['age']['type'] === 'integer'; + }); +}); + +test('it can stream with structured output', function (): void { + $llm = AnthropicChat::fake([ + ChatCreateStreamedResponse::fake(fopen(__DIR__ . '/../../../fixtures/anthropic/chat-stream-json.txt', 'r')), + ], 'claude-3-5-sonnet-20241022'); + + $llm->addFeature(ModelFeature::StructuredOutput); + $llm->withStreaming(); + + class AnthropicJoke + { + public function __construct( + public ?string $setup = null, + public ?string $punchline = null, + ) {} + } + + $result = $llm->withStructuredOutput(AnthropicJoke::class)->invoke([ + new UserMessage('Tell me a joke about dogs'), + ]); + + expect($result)->toBeInstanceOf(ChatStreamResult::class); + + $hasValidParsedOutput = false; + + $result->each(function (ChatGenerationChunk $chunk) use (&$hasValidParsedOutput): void { + expect($chunk)->toBeInstanceOf(ChatGenerationChunk::class); + + // Check if this chunk has the parsed output we expect + if ($chunk->isFinal) { + expect($chunk->parsedOutput->setup)->toBe('Why did the dog sit in the shade?') + ->and($chunk->parsedOutput->punchline)->toBe("Because he didn't want to be a hot dog!"); + + $hasValidParsedOutput = true; + } + }); + + expect($hasValidParsedOutput)->toBeTrue('Expected at least one chunk to have valid parsed output'); + + /** @var \Anthropic\Testing\ClientFake $client */ + $client = $llm->getClient(); + + $client->messages()->assertSent(function (string $method, array $parameters): bool { + return $parameters['model'] === 'claude-3-5-sonnet-20241022' + && $parameters['messages'][0]['role'] === 'user' + && $parameters['messages'][0]['content'] === 'Tell me a joke about dogs'; + }); +}); + +test('it can stream with tool calls', function (): void { + $llm = AnthropicChat::fake([ + ChatCreateStreamedResponse::fake(fopen(__DIR__ . '/../../../fixtures/anthropic/chat-stream-tool-calls.txt', 'r')), + ], 'claude-3-5-sonnet-20241022'); + + $llm->addFeature(ModelFeature::ToolCalling); + $llm->withStreaming(); + + $llm->withTools([ + #[Tool(name: 'multiply', description: 'Multiply two numbers')] + fn(int $x, int $y): int => $x * $y, + ]); + + $result = $llm->invoke([ + new UserMessage('What is 3 times 4?'), + ]); + + expect($result)->toBeInstanceOf(ChatStreamResult::class); + + $hasToolCalls = false; + $toolCallsAccumulated = false; + + $result->each(function (ChatGenerationChunk $chunk) use (&$hasToolCalls, &$toolCallsAccumulated): void { + expect($chunk)->toBeInstanceOf(ChatGenerationChunk::class) + ->and($chunk->message)->toBeInstanceOf(AssistantMessage::class); + + // Check if we have tool calls in any chunk + if ($chunk->message->toolCalls?->isNotEmpty()) { + $hasToolCalls = true; + + // Check if tool calls are properly accumulated + if ($chunk->isFinal) { + expect($chunk->message->toolCalls) + ->toBeInstanceOf(ToolCallCollection::class) + ->and($chunk->message->toolCalls) + ->toHaveCount(1) + ->and($chunk->message->toolCalls[0]) + ->toBeInstanceOf(ToolCall::class) + ->and($chunk->message->toolCalls[0]->id) + ->toBe('call_multiply_123') + ->and($chunk->message->toolCalls[0]->function) + ->toBeInstanceOf(FunctionCall::class) + ->and($chunk->message->toolCalls[0]->function->name) + ->toBe('multiply') + ->and($chunk->message->toolCalls[0]->function->arguments) + ->toBe([ + 'x' => 3, + 'y' => 4, + ]); + + $toolCallsAccumulated = true; + } + } + }); + + expect($hasToolCalls)->toBeTrue('Expected at least one chunk to have tool calls') + ->and($toolCallsAccumulated)->toBeTrue('Expected final chunk to have complete tool call data'); + + /** @var \Anthropic\Testing\ClientFake $client */ + $client = $llm->getClient(); + + $client->messages()->assertSent(function (string $method, array $parameters): bool { + return $parameters['model'] === 'claude-3-5-sonnet-20241022' + && $parameters['messages'][0]['role'] === 'user' + && $parameters['messages'][0]['content'] === 'What is 3 times 4?' + && $parameters['tools'][0]['type'] === 'custom' + && $parameters['tools'][0]['name'] === 'multiply'; + }); +}); + +test('it can use structured output with an enum', function (): void { + $llm = AnthropicChat::fake([ + createAnthropicResponse([ + 'content' => [ + [ + 'type' => 'text', + 'text' => '{"AnthropicSentiment":"positive"}', + ], + ], + ]), + createAnthropicResponse([ + 'content' => [ + [ + 'type' => 'text', + 'text' => '{"AnthropicSentiment":"neutral"}', + ], + ], + ]), + createAnthropicResponse([ + 'content' => [ + [ + 'type' => 'text', + 'text' => '{"AnthropicSentiment":"negative"}', + ], + ], + ]), + createAnthropicResponse([ + 'content' => [ + [ + 'type' => 'text', + 'text' => '{"AnthropicSentiment":"neutral"}', + ], + ], + ]), + ], 'claude-3-5-sonnet-20241022'); + + $llm->addFeature(ModelFeature::StructuredOutput); + + enum AnthropicSentiment: string + { + case Positive = 'positive'; + case Negative = 'negative'; + case Neutral = 'neutral'; + } + + $llm->withStructuredOutput(AnthropicSentiment::class); + + expect($llm->invoke('Analyze the sentiment of this text: This pizza is awesome')->parsedOutput) + ->toBe(AnthropicSentiment::Positive); + + expect($llm->invoke('Analyze the sentiment of this text: This pizza is okay')->parsedOutput) + ->toBe(AnthropicSentiment::Neutral); + + expect($llm->invoke('Analyze the sentiment of this text: This pizza is terrible')->parsedOutput) + ->toBe(AnthropicSentiment::Negative); + + $getSentiment = Cortex::prompt('Analyze the sentiment of this text: {input}')->llm($llm); + + $result = $getSentiment->invoke('This pizza is average'); + + expect($result)->toBeInstanceOf(ChatResult::class) + ->and($result->parsedOutput)->toBe(AnthropicSentiment::Neutral); +}); + +test('it can force json output', function (): void { + $response = createAnthropicResponse([ + 'content' => [ + [ + 'type' => 'text', + 'text' => '{"setup":"Why did the scarecrow win an award?","punchline":"Because he was outstanding in his field!"}', + ], + ], + ]); + + $llm = AnthropicChat::fake([ + $response, + ], 'claude-3-5-sonnet-20241022'); + + $llm->addFeature(ModelFeature::JsonOutput); + $llm->forceJsonOutput(); + + $result = $llm->invoke([ + new UserMessage('Tell me a joke'), + ]); + + expect($result->generation->message->content) + ->toBeArray() + ->toContainOnlyInstancesOf(TextContent::class) + ->and($result->generation->message->content[0]->text) + ->toBe('{"setup":"Why did the scarecrow win an award?","punchline":"Because he was outstanding in his field!"}'); +}); + +test('it can set temperature and max tokens', function (): void { + $llm = AnthropicChat::fake([ + createAnthropicResponse([ + 'content' => [ + [ + 'type' => 'text', + 'text' => 'Hello!', + ], + ], + ]), + ]); + + $llm->withTemperature(0.7)->withMaxTokens(100)->invoke([ + new UserMessage('Hello'), + ]); + + /** @var \Anthropic\Testing\ClientFake $client */ + $client = $llm->getClient(); + + $client->messages()->assertSent(function (string $method, array $parameters): bool { + return $parameters['temperature'] === 0.7 && $parameters['max_tokens'] === 100; + }); +}); + +test('it tracks token usage', function (): void { + $response = createAnthropicResponse([ + 'usage' => [ + 'input_tokens' => 10, + 'output_tokens' => 20, + ], + 'content' => [ + [ + 'type' => 'text', + 'text' => 'Hello!', + ], + ], + ]); + + $llm = AnthropicChat::fake([ + $response, + ]); + + $result = $llm->invoke([ + new UserMessage('Hello'), + ]); + + expect($result->usage) + ->toBeInstanceOf(Usage::class) + ->and($result->usage->promptTokens)->toBe(10) + ->and($result->usage->completionTokens)->toBe(20); +}); + +// Helper function to create Anthropic responses without fake() merging issues +function createAnthropicResponse(array $attributes): ChatCreateResponse +{ + $defaults = [ + 'id' => 'msg_test_' . uniqid(), + 'type' => 'message', + 'role' => 'assistant', + 'model' => 'claude-3-5-sonnet-20241022', + 'stop_sequence' => null, + 'stop_reason' => 'end_turn', + 'usage' => [ + 'input_tokens' => 10, + 'output_tokens' => 20, + ], + ]; + + return ChatCreateResponse::from( + array_merge($defaults, $attributes), + MetaInformation::from([]), + ); +} diff --git a/tests/Unit/LLM/Drivers/OpenAIChatTest.php b/tests/Unit/LLM/Drivers/OpenAIChatTest.php index f1d1ed2..479fc69 100644 --- a/tests/Unit/LLM/Drivers/OpenAIChatTest.php +++ b/tests/Unit/LLM/Drivers/OpenAIChatTest.php @@ -19,6 +19,7 @@ use Cortex\LLM\Data\Messages\UserMessage; use Cortex\Tasks\Enums\StructuredOutputMode; use Cortex\LLM\Data\Messages\AssistantMessage; +use Cortex\LLM\Data\Messages\Content\TextContent; use OpenAI\Responses\Chat\CreateResponse as ChatCreateResponse; use OpenAI\Responses\Chat\CreateStreamedResponse as ChatCreateStreamedResponse; @@ -44,7 +45,7 @@ ->and($result->rawResponse)->toBeArray()->not->toBeEmpty() ->and($result->generations)->toHaveCount(1) ->and($result->generation->message)->toBeInstanceOf(AssistantMessage::class) - ->and($result->generation->message->content)->toBe('I am doing well, thank you for asking!'); + ->and($result->generation->message->text())->toBe('I am doing well, thank you for asking!'); }); test('it can stream', function (): void { @@ -152,8 +153,11 @@ new UserMessage('Tell me about a person'), ]); - expect($result->generation->message->content) - ->toBe('{"name":"John Doe","age":30}'); + expect($result->generation->message->text()) + ->toBe('{"name":"John Doe","age":30}') + ->and($result->generation->message->content()) + ->toBeArray() + ->toContainOnlyInstancesOf(TextContent::class); expect($result->generation->parsedOutput)->toBe([ 'name' => 'John Doe', @@ -373,8 +377,11 @@ enum Sentiment: string new UserMessage('Tell me a joke'), ]); - expect($result->generation->message->content) - ->toBe('{"setup":"Why did the scarecrow win an award?","punchline":"Because he was outstanding in his field!"}'); + expect($result->generation->message->text()) + ->toBe('{"setup":"Why did the scarecrow win an award?","punchline":"Because he was outstanding in his field!"}') + ->and($result->generation->message->content()) + ->toBeArray() + ->toContainOnlyInstancesOf(TextContent::class); }); test('it can set temperature and max tokens', function (): void { diff --git a/tests/Unit/Prompts/Factories/LangfusePromptFactoryTest.php b/tests/Unit/Prompts/Factories/LangfusePromptFactoryTest.php index f6ddb36..cdf7296 100644 --- a/tests/Unit/Prompts/Factories/LangfusePromptFactoryTest.php +++ b/tests/Unit/Prompts/Factories/LangfusePromptFactoryTest.php @@ -16,6 +16,7 @@ use Cortex\LLM\Data\Messages\UserMessage; use Cortex\LLM\Data\Messages\SystemMessage; use Cortex\Prompts\Contracts\PromptTemplate; +use Cortex\Tasks\Enums\StructuredOutputMode; use Cortex\LLM\Data\Messages\AssistantMessage; use Cortex\LLM\Data\Messages\MessagePlaceholder; use Cortex\Prompts\Templates\ChatPromptTemplate; @@ -258,7 +259,7 @@ function createLangfuseFactory( 'max_tokens' => 100, ], 'tools' => ['calculator', 'search'], - 'structuredOutput' => [ + 'structured_output' => [ 'type' => 'object', 'properties' => [ 'name' => [ @@ -267,6 +268,7 @@ function createLangfuseFactory( ], 'required' => ['name'], ], + 'structured_output_mode' => StructuredOutputMode::Tool->value, 'additional' => [ 'custom_field' => 'custom_value', ], @@ -294,6 +296,7 @@ function createLangfuseFactory( expect($prompt->metadata->tools)->toBe(['calculator', 'search']); expect($prompt->metadata->structuredOutput)->toBeInstanceOf(ObjectSchema::class); expect($prompt->metadata->structuredOutput->getPropertyKeys())->toBe(['name']); + expect($prompt->metadata->structuredOutputMode)->toBe(StructuredOutputMode::Tool); expect($prompt->metadata->additional)->toBe([ 'custom_field' => 'custom_value', ]); @@ -395,6 +398,7 @@ function createLangfuseFactory( ], tools: [], structuredOutput: null, + structuredOutputMode: StructuredOutputMode::Auto, additional: [ 'extra' => $config['extra_data'] ?? null, ], @@ -411,6 +415,7 @@ function createLangfuseFactory( ]); expect($prompt->metadata->tools)->toBe([]); expect($prompt->metadata->structuredOutput)->toBeNull(); + expect($prompt->metadata->structuredOutputMode)->toBe(StructuredOutputMode::Auto); expect($prompt->metadata->additional)->toBe([ 'extra' => 'some value', ]); diff --git a/tests/Unit/Prompts/Templates/ChatPromptTemplateTest.php b/tests/Unit/Prompts/Templates/ChatPromptTemplateTest.php index b23917c..7a2c753 100644 --- a/tests/Unit/Prompts/Templates/ChatPromptTemplateTest.php +++ b/tests/Unit/Prompts/Templates/ChatPromptTemplateTest.php @@ -33,7 +33,7 @@ expect($messages)->toBeInstanceOf(MessageCollection::class); expect($messages)->toHaveCount(1); - expect($messages[0]->content())->toBe('Hello, my name is John!'); + expect($messages[0]->text())->toBe('Hello, my name is John!'); }); test('it can format a chat prompt template with multiple variables and messages', function (): void { @@ -190,7 +190,7 @@ ]); expect($result)->toBeInstanceOf(ChatResult::class); - expect($result->generation->message->content)->toBe("Hello John! It's nice to meet you."); + expect($result->generation->message->text())->toBe("Hello John! It's nice to meet you."); /** @var \OpenAI\Testing\ClientFake $client */ $client = $llm->getClient(); diff --git a/tests/fixtures/anthropic/chat-stream-json.txt b/tests/fixtures/anthropic/chat-stream-json.txt new file mode 100644 index 0000000..b47a7ba --- /dev/null +++ b/tests/fixtures/anthropic/chat-stream-json.txt @@ -0,0 +1,32 @@ +event: message_start +data: {"type": "message_start", "message": {"id": "msg_test", "type": "message", "role": "assistant", "content": [], "model": "claude-3-5-sonnet-20241022", "stop_reason": null, "stop_sequence": null, "usage": {"input_tokens": 25, "output_tokens": 1}}} + +event: content_block_start +data: {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}} + +event: content_block_delta +data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "{"}} + +event: content_block_delta +data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "\"setup\":"}} + +event: content_block_delta +data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "\"Why did the dog sit in the shade?\""}} + +event: content_block_delta +data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": ",\"punchline\":"}} + +event: content_block_delta +data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "\"Because he didn't want to be a hot dog!\""}} + +event: content_block_delta +data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "}"}} + +event: content_block_stop +data: {"type": "content_block_stop", "index": 0} + +event: message_delta +data: {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence":null}, "usage": {"output_tokens": 30}} + +event: message_stop +data: {"type": "message_stop"} diff --git a/tests/fixtures/anthropic/chat-stream-tool-calls.txt b/tests/fixtures/anthropic/chat-stream-tool-calls.txt new file mode 100644 index 0000000..fa623ba --- /dev/null +++ b/tests/fixtures/anthropic/chat-stream-tool-calls.txt @@ -0,0 +1,29 @@ +event: message_start +data: {"type": "message_start", "message": {"id": "msg_test_tool", "type": "message", "role": "assistant", "content": [], "model": "claude-3-5-sonnet-20241022", "stop_reason": null, "stop_sequence": null, "usage": {"input_tokens": 25, "output_tokens": 1}}} + +event: content_block_start +data: {"type": "content_block_start", "index": 0, "content_block": {"type": "tool_use", "id": "call_multiply_123", "name": "multiply", "input": {}}} + +event: content_block_delta +data: {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": "{\"x\":"}} + +event: content_block_delta +data: {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": "3"}} + +event: content_block_delta +data: {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": ",\"y\":"}} + +event: content_block_delta +data: {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": "4"}} + +event: content_block_delta +data: {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": "}"}} + +event: content_block_stop +data: {"type": "content_block_stop", "index": 0} + +event: message_delta +data: {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence":null}, "usage": {"output_tokens": 25}} + +event: message_stop +data: {"type": "message_stop"}