diff --git a/composer.json b/composer.json index e2356c0..c8d74cb 100644 --- a/composer.json +++ b/composer.json @@ -23,6 +23,10 @@ "guzzlehttp/psr7": "^2.8", "illuminate/collections": "^12.49", "laravel/prompts": "^0.3.8", + "open-telemetry/api": "^1.8", + "open-telemetry/exporter-otlp": "^1.4", + "open-telemetry/sdk": "^1.13", + "open-telemetry/sem-conv": "^1.38", "openai-php/client": "^0.18", "php-mcp/client": "^1.0", "psr-discovery/cache-implementations": "^1.2", @@ -104,7 +108,8 @@ "allow-plugins": { "dealerdirect/phpcodesniffer-composer-installer": true, "pestphp/pest-plugin": true, - "php-http/discovery": true + "php-http/discovery": true, + "tbachert/spi": true } }, "extra": { diff --git a/config/cortex.php b/config/cortex.php index 9744c2a..4ab5010 100644 --- a/config/cortex.php +++ b/config/cortex.php @@ -5,6 +5,7 @@ use Cortex\LLM\Enums\LLMDriver; use Cortex\LLM\Enums\StreamingProtocol; use Cortex\Agents\Prebuilt\WeatherAgent; +use OpenTelemetry\Contrib\Otlp\Protocols; use Cortex\ModelInfo\Providers\OllamaModelInfoProvider; use Cortex\ModelInfo\Providers\LiteLLMModelInfoProvider; use Cortex\ModelInfo\Providers\LMStudioModelInfoProvider; @@ -367,4 +368,45 @@ | */ 'default_streaming_protocol' => StreamingProtocol::Vercel, + + /* + |-------------------------------------------------------------------------- + | OpenTelemetry Tracing + |-------------------------------------------------------------------------- + | + | Configure OpenTelemetry tracing to export spans for agent runs, LLM calls, + | tool executions, and agent steps to any OTLP-compatible backend + | (e.g. Jaeger, Grafana Tempo, Honeycomb, Datadog). + | + | Set CORTEX_TRACING_ENABLED=true and point OTEL_EXPORTER_OTLP_ENDPOINT + | at your collector or backend to get started. + | + */ + 'tracing' => [ + 'enabled' => env('CORTEX_TRACING_ENABLED', false), + + 'exporter' => [ + /** + * The full OTLP traces endpoint URL. + * For HTTP/protobuf (default): http://localhost:4318/v1/traces + * For gRPC: http://localhost:4317 + */ + 'endpoint' => env('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://localhost:4318/v1/traces'), + + /** + * Supported: "http/protobuf", "http/json" + */ + 'protocol' => env('OTEL_EXPORTER_OTLP_PROTOCOL', Protocols::HTTP_PROTOBUF), + + /** + * The headers to send with the request. Comma separated list of key=value pairs. + */ + 'headers' => env('OTEL_EXPORTER_OTLP_HEADERS', ''), + ], + + /** + * The service name reported to the tracing backend. + */ + 'service_name' => env('OTEL_SERVICE_NAME', 'cortex'), + ], ]; diff --git a/src/Agents/Middleware/AfterModelWrapper.php b/src/Agents/Middleware/AfterModelWrapper.php index 37c7cc4..5dadcb6 100644 --- a/src/Agents/Middleware/AfterModelWrapper.php +++ b/src/Agents/Middleware/AfterModelWrapper.php @@ -5,8 +5,14 @@ namespace Cortex\Agents\Middleware; use Closure; +use Throwable; +use Cortex\Events\MiddlewareEnd; +use Cortex\Events\MiddlewareError; +use Cortex\Events\MiddlewareStart; use Cortex\Pipeline\RuntimeConfig; use Cortex\Support\Traits\CanPipe; +use Cortex\Support\Traits\DispatchesEvents; +use Cortex\Events\Contracts\MiddlewareEvent; use Cortex\Agents\Contracts\AfterModelMiddleware; /** @@ -16,6 +22,7 @@ class AfterModelWrapper implements AfterModelMiddleware { use CanPipe; + use DispatchesEvents; public function __construct( protected AfterModelMiddleware $middleware, @@ -23,11 +30,28 @@ public function __construct( public function handlePipeable(mixed $payload, RuntimeConfig $config, Closure $next): mixed { - return $this->middleware->afterModel($payload, $config, $next); + $this->dispatchEvent(new MiddlewareStart($this->middleware, $config, 'afterModel')); + + try { + $result = $this->middleware->afterModel($payload, $config, $next); + } catch (Throwable $e) { + $this->dispatchEvent(new MiddlewareError($this->middleware, $config, 'afterModel', $e)); + + throw $e; + } + + $this->dispatchEvent(new MiddlewareEnd($this->middleware, $config, 'afterModel')); + + return $result; } public function afterModel(mixed $payload, RuntimeConfig $config, Closure $next): mixed { return $this->handlePipeable($payload, $config, $next); } + + protected function eventBelongsToThisInstance(object $event): bool + { + return $event instanceof MiddlewareEvent && $event->middleware === $this->middleware; + } } diff --git a/src/Agents/Middleware/BeforeModelWrapper.php b/src/Agents/Middleware/BeforeModelWrapper.php index 71e9fae..ece7b26 100644 --- a/src/Agents/Middleware/BeforeModelWrapper.php +++ b/src/Agents/Middleware/BeforeModelWrapper.php @@ -5,8 +5,14 @@ namespace Cortex\Agents\Middleware; use Closure; +use Throwable; +use Cortex\Events\MiddlewareEnd; +use Cortex\Events\MiddlewareError; +use Cortex\Events\MiddlewareStart; use Cortex\Pipeline\RuntimeConfig; use Cortex\Support\Traits\CanPipe; +use Cortex\Support\Traits\DispatchesEvents; +use Cortex\Events\Contracts\MiddlewareEvent; use Cortex\Agents\Contracts\BeforeModelMiddleware; /** @@ -16,6 +22,7 @@ class BeforeModelWrapper implements BeforeModelMiddleware { use CanPipe; + use DispatchesEvents; public function __construct( protected BeforeModelMiddleware $middleware, @@ -23,11 +30,28 @@ public function __construct( public function handlePipeable(mixed $payload, RuntimeConfig $config, Closure $next): mixed { - return $this->middleware->beforeModel($payload, $config, $next); + $this->dispatchEvent(new MiddlewareStart($this->middleware, $config, 'beforeModel')); + + try { + $result = $this->middleware->beforeModel($payload, $config, $next); + } catch (Throwable $e) { + $this->dispatchEvent(new MiddlewareError($this->middleware, $config, 'beforeModel', $e)); + + throw $e; + } + + $this->dispatchEvent(new MiddlewareEnd($this->middleware, $config, 'beforeModel')); + + return $result; } public function beforeModel(mixed $payload, RuntimeConfig $config, Closure $next): mixed { return $this->handlePipeable($payload, $config, $next); } + + protected function eventBelongsToThisInstance(object $event): bool + { + return $event instanceof MiddlewareEvent && $event->middleware === $this->middleware; + } } diff --git a/src/Agents/Middleware/BeforePromptWrapper.php b/src/Agents/Middleware/BeforePromptWrapper.php index bf5c9a9..93fdfa6 100644 --- a/src/Agents/Middleware/BeforePromptWrapper.php +++ b/src/Agents/Middleware/BeforePromptWrapper.php @@ -5,8 +5,14 @@ namespace Cortex\Agents\Middleware; use Closure; +use Throwable; +use Cortex\Events\MiddlewareEnd; +use Cortex\Events\MiddlewareError; +use Cortex\Events\MiddlewareStart; use Cortex\Pipeline\RuntimeConfig; use Cortex\Support\Traits\CanPipe; +use Cortex\Support\Traits\DispatchesEvents; +use Cortex\Events\Contracts\MiddlewareEvent; use Cortex\Agents\Contracts\BeforePromptMiddleware; /** @@ -16,6 +22,7 @@ class BeforePromptWrapper implements BeforePromptMiddleware { use CanPipe; + use DispatchesEvents; public function __construct( protected BeforePromptMiddleware $middleware, @@ -23,11 +30,28 @@ public function __construct( public function handlePipeable(mixed $payload, RuntimeConfig $config, Closure $next): mixed { - return $this->middleware->beforePrompt($payload, $config, $next); + $this->dispatchEvent(new MiddlewareStart($this->middleware, $config, 'beforePrompt')); + + try { + $result = $this->middleware->beforePrompt($payload, $config, $next); + } catch (Throwable $e) { + $this->dispatchEvent(new MiddlewareError($this->middleware, $config, 'beforePrompt', $e)); + + throw $e; + } + + $this->dispatchEvent(new MiddlewareEnd($this->middleware, $config, 'beforePrompt')); + + return $result; } public function beforePrompt(mixed $payload, RuntimeConfig $config, Closure $next): mixed { return $this->handlePipeable($payload, $config, $next); } + + protected function eventBelongsToThisInstance(object $event): bool + { + return $event instanceof MiddlewareEvent && $event->middleware === $this->middleware; + } } diff --git a/src/Agents/Prebuilt/WeatherAgent.php b/src/Agents/Prebuilt/WeatherAgent.php index d16eca6..d3058e2 100644 --- a/src/Agents/Prebuilt/WeatherAgent.php +++ b/src/Agents/Prebuilt/WeatherAgent.php @@ -55,7 +55,8 @@ public function prompt(): ChatPromptTemplate|ChatPromptBuilder|string public function llm(): LLM|string|null { - return Cortex::llm('ollama', 'qwen3.5:9b')->ignoreFeatures(); + return Cortex::llm('lmstudio/openai/gpt-oss-20b')->ignoreFeatures(); + // return Cortex::llm('ollama', 'qwen3.5:9b')->ignoreFeatures(); // return Cortex::llm('lmstudio/qwen3.5-9b-mlx')->ignoreFeatures(); // return Cortex::llm('anthropic', 'claude-haiku-4-5')->ignoreFeatures(); // return Cortex::llm('openai', 'gpt-5-mini')->ignoreFeatures(); diff --git a/src/Agents/Registry.php b/src/Agents/Registry.php index eb06da8..36ee430 100644 --- a/src/Agents/Registry.php +++ b/src/Agents/Registry.php @@ -29,7 +29,6 @@ public function register(Agent|string $agent, ?string $idOverride = null): void ); } - // @phpstan-ignore function.alreadyNarrowedType if (! is_subclass_of($agent, AbstractAgentBuilder::class)) { throw new InvalidArgumentException( sprintf( diff --git a/src/CortexServiceProvider.php b/src/CortexServiceProvider.php index 77acb85..3cc7861 100644 --- a/src/CortexServiceProvider.php +++ b/src/CortexServiceProvider.php @@ -5,25 +5,31 @@ namespace Cortex; use Throwable; -use Monolog\Logger; use Cortex\LLM\LLMManager; use Cortex\Agents\Registry; use Cortex\Console\AgentChat; use Cortex\LLM\Contracts\LLM; use Cortex\Mcp\McpServerManager; -use Monolog\Handler\StreamHandler; -use Monolog\Formatter\LineFormatter; use Illuminate\Support\Facades\Blade; use Cortex\ModelInfo\ModelInfoFactory; use Spatie\LaravelPackageTools\Package; use Cortex\Embeddings\EmbeddingsManager; use Cortex\Prompts\PromptFactoryManager; +use OpenTelemetry\Contrib\Otlp\Protocols; use Cortex\Embeddings\Contracts\Embeddings; use Cortex\Prompts\Contracts\PromptFactory; +use OpenTelemetry\Contrib\Otlp\SpanExporter; +use OpenTelemetry\SDK\Resource\ResourceInfo; use Illuminate\Contracts\Container\Container; use Cortex\Support\Events\InternalEventDispatcher; +use OpenTelemetry\SDK\Common\Attribute\Attributes; +use OpenTelemetry\SDK\Common\Util\ShutdownHandler; +use OpenTelemetry\SDK\Trace\TracerProviderBuilder; +use OpenTelemetry\SDK\Resource\ResourceInfoFactory; use Spatie\LaravelPackageTools\PackageServiceProvider; -use Cortex\Support\Events\Subscribers\LoggingSubscriber; +use OpenTelemetry\Contrib\Otlp\OtlpHttpTransportFactory; +use OpenTelemetry\SDK\Trace\SpanProcessor\SimpleSpanProcessor; +use Cortex\Support\Events\Subscribers\Otel\OpenTelemetrySubscriber; class CortexServiceProvider extends PackageServiceProvider { @@ -55,6 +61,7 @@ public function packageBooted(): void $this->registerBladeDirectives(); $this->setupLogging(); + $this->setupTracing(); } protected function registerBladeDirectives(): void @@ -163,12 +170,67 @@ protected function setupLogging(): void } // TODO: This will be configurable. - $logger = new Logger('cortex'); - $handler = new StreamHandler('php://stdout'); - $handler->setFormatter(new LineFormatter()); + // $logger = new Logger('cortex'); + // $handler = new StreamHandler('php://stdout'); + // $handler->setFormatter(new LineFormatter()); - $logger->pushHandler($handler); + // $logger->pushHandler($handler); - InternalEventDispatcher::instance()->subscribe(new LoggingSubscriber($logger)); + // InternalEventDispatcher::instance()->subscribe(new LoggingSubscriber($logger)); + } + + protected function setupTracing(): void + { + if ($this->app->runningUnitTests()) { + return; + } + + if (! config('cortex.tracing.enabled', false)) { + return; + } + + $endpoint = (string) config('cortex.tracing.exporter.endpoint', 'http://localhost:4318/v1/traces'); + $protocol = (string) config('cortex.tracing.exporter.protocol', Protocols::HTTP_PROTOBUF); + $serviceName = (string) config('cortex.tracing.service_name', 'cortex'); + $rawHeaders = config('cortex.tracing.exporter.headers', ''); + $headers = []; + + foreach (explode(',', $rawHeaders) as $header) { + $header = trim($header); + + if ($header === '') { + continue; + } + + $parts = explode('=', $header, 2); + + if (count($parts) === 2) { + [$key, $value] = $parts; + $headers[trim($key)] = trim($value, " \t\n\r\0\x0B\"'"); + } + } + + $resource = ResourceInfoFactory::emptyResource()->merge( + ResourceInfo::create(Attributes::create([ + 'service.name' => $serviceName, + ])), + ); + + $transport = new OtlpHttpTransportFactory()->create( + $endpoint, + Protocols::contentType($protocol), + $headers, + ); + + $exporter = new SpanExporter($transport); + + $tracerProvider = new TracerProviderBuilder() + ->addSpanProcessor(new SimpleSpanProcessor($exporter)) + ->setResource($resource) + ->build(); + + ShutdownHandler::register($tracerProvider->shutdown(...)); + + InternalEventDispatcher::instance()->subscribe(new OpenTelemetrySubscriber($tracerProvider)); } } diff --git a/src/Events/Contracts/MiddlewareEvent.php b/src/Events/Contracts/MiddlewareEvent.php new file mode 100644 index 0000000..2709aad --- /dev/null +++ b/src/Events/Contracts/MiddlewareEvent.php @@ -0,0 +1,15 @@ + $this->config->runId, + 'middleware' => $this->middleware::class, + 'hook' => $this->hook, + ]; + } +} diff --git a/src/Events/MiddlewareError.php b/src/Events/MiddlewareError.php new file mode 100644 index 0000000..f2d5195 --- /dev/null +++ b/src/Events/MiddlewareError.php @@ -0,0 +1,35 @@ + $this->config->runId, + 'middleware' => $this->middleware::class, + 'hook' => $this->hook, + 'exception' => $this->exception->getMessage(), + ]; + } +} diff --git a/src/Events/MiddlewareStart.php b/src/Events/MiddlewareStart.php new file mode 100644 index 0000000..3e2d86b --- /dev/null +++ b/src/Events/MiddlewareStart.php @@ -0,0 +1,32 @@ + $this->config->runId, + 'middleware' => $this->middleware::class, + 'hook' => $this->hook, + ]; + } +} diff --git a/src/LLM/AbstractLLM.php b/src/LLM/AbstractLLM.php index d5f47c4..988083b 100644 --- a/src/LLM/AbstractLLM.php +++ b/src/LLM/AbstractLLM.php @@ -412,6 +412,11 @@ public function getMaxTokens(): ?int return $this->maxTokens; } + public function getToolConfig(): ?ToolConfig + { + return $this->toolConfig; + } + public function isStreaming(): bool { return $this->streaming; diff --git a/src/LLM/Contracts/LLM.php b/src/LLM/Contracts/LLM.php index 8d18566..a6c3a6e 100644 --- a/src/LLM/Contracts/LLM.php +++ b/src/LLM/Contracts/LLM.php @@ -7,6 +7,7 @@ use Closure; use Cortex\Contracts\Pipeable; use Cortex\LLM\Data\ChatResult; +use Cortex\LLM\Data\ToolConfig; use Cortex\LLM\Enums\ToolChoice; use Cortex\ModelInfo\Data\ModelInfo; use Cortex\LLM\Data\ChatStreamResult; @@ -192,6 +193,11 @@ public function getStructuredOutputConfig(): ?StructuredOutputConfig; */ public function getStructuredOutputMode(): StructuredOutputMode; + /** + * Get the tool config for the LLM. + */ + public function getToolConfig(): ?ToolConfig; + /** * Set whether the raw provider response should be included in the result, if available. */ diff --git a/src/Support/Events/Subscribers/Otel/Concerns/SerializesMessages.php b/src/Support/Events/Subscribers/Otel/Concerns/SerializesMessages.php new file mode 100644 index 0000000..52da01c --- /dev/null +++ b/src/Support/Events/Subscribers/Otel/Concerns/SerializesMessages.php @@ -0,0 +1,139 @@ +withoutPlaceholders()->map( + fn(Message $message): array => $this->serializeMessage($message), + )->values()->all(); + + return (string) json_encode($serialized); + } + + /** + * @return array + */ + private function serializeMessage(Message $message): array + { + $result = [ + 'role' => $message->role()->value, + 'parts' => $this->serializeContentToParts($message), + ]; + + if (property_exists($message, 'name') && $message->name !== null) { + $result['name'] = $message->name; + } + + return $result; + } + + /** + * @return array> + */ + private function serializeContentToParts(Message $message): array + { + $content = $message->content(); + + // Tool messages are always a single tool_call_response part + if (property_exists($message, 'id') && $message->role()->value === 'tool') { + return [[ + 'type' => 'tool_call_response', + 'id' => $message->id, + 'response' => is_string($content) ? $content : (string) json_encode($content), + ]]; + } + + $parts = []; + + if (is_string($content) && $content !== '') { + $parts[] = [ + 'type' => 'text', + 'content' => $content, + ]; + } elseif (is_array($content)) { + foreach ($content as $item) { + $part = match (true) { + $item instanceof TextContent => $item->text !== null + ? [ + 'type' => 'text', + 'content' => $item->text, + ] + : null, + $item instanceof ReasoningContent => [ + 'type' => 'reasoning', + 'content' => $item->reasoning, + ], + $item instanceof ImageContent => $this->serializeImagePart($item), + $item instanceof AudioContent => [ + 'type' => 'blob', + 'modality' => 'audio', + 'mime_type' => 'audio/' . $item->format, + 'content' => $item->base64Data, + ], + default => null, + }; + + if ($part !== null) { + $parts[] = $part; + } + } + } + + // Tool call requests live on assistant messages alongside any text content + if (property_exists($message, 'toolCalls') && $message->toolCalls !== null) { + foreach ($message->toolCalls as $toolCall) { + /** @var ToolCall $toolCall */ + $parts[] = [ + 'type' => 'tool_call', + 'id' => $toolCall->id, + 'name' => $toolCall->function->name, + 'arguments' => $toolCall->function->arguments, + ]; + } + } + + return $parts; + } + + /** + * @return array + */ + private function serializeImagePart(ImageContent $image): array + { + $url = $image->url; + + if (str_starts_with($url, 'data:')) { + return [ + 'type' => 'blob', + 'modality' => 'image', + 'mime_type' => $image->mimeType ?? 'image/jpeg', + 'content' => (string) preg_replace('/^data:[^;]+;base64,/', '', $url), + ]; + } + + return [ + 'type' => 'uri', + 'modality' => 'image', + 'mime_type' => $image->mimeType, + 'uri' => $url, + ]; + } +} diff --git a/src/Support/Events/Subscribers/Otel/Concerns/TracksAgentSpans.php b/src/Support/Events/Subscribers/Otel/Concerns/TracksAgentSpans.php new file mode 100644 index 0000000..96e37f1 --- /dev/null +++ b/src/Support/Events/Subscribers/Otel/Concerns/TracksAgentSpans.php @@ -0,0 +1,77 @@ + + */ + private array $agentSpans = []; + + private function registerAgentListeners(InternalEventDispatcher $dispatcher, TracerInterface $tracer): void + { + $dispatcher->listen(AgentStart::class, function (AgentStart $event) use ($tracer): void { + $runId = $event->config->runId ?? 'default'; + $agentName = $event->agent->getName(); + + $spanName = $agentName !== null ? 'invoke_agent ' . $agentName : 'invoke_agent'; + + $span = $tracer->spanBuilder($spanName) + ->setSpanKind(SpanKind::KIND_INTERNAL) + ->startSpan(); + + $span->setAttribute(GenAiAttributes::OPERATION_NAME, 'invoke_agent'); + $span->setAttribute(GenAiAttributes::AGENT_ID, $event->agent->getId()); + + if ($agentName !== null) { + $span->setAttribute(GenAiAttributes::AGENT_NAME, $agentName); + } + + $agentDescription = $event->agent->getDescription(); + + if ($agentDescription !== null) { + $span->setAttribute(GenAiAttributes::AGENT_DESCRIPTION, $agentDescription); + } + + if ($event->config?->runId !== null) { + $span->setAttribute('cortex.run_id', $event->config->runId); + } + + if ($event->config?->threadId !== null) { + $span->setAttribute(GenAiAttributes::CONVERSATION_ID, $event->config->threadId); + } + + $this->agentSpans[$runId] = [ + 'span' => $span, + 'scope' => $span->activate(), + ]; + }); + + $dispatcher->listen(AgentEnd::class, function (AgentEnd $event): void { + $runId = $event->config->runId ?? 'default'; + + if (! isset($this->agentSpans[$runId])) { + return; + } + + ['span' => $span, 'scope' => $scope] = $this->agentSpans[$runId]; + + $scope->detach(); + $span->end(); + + unset($this->agentSpans[$runId]); + }); + } +} diff --git a/src/Support/Events/Subscribers/Otel/Concerns/TracksLlmSpans.php b/src/Support/Events/Subscribers/Otel/Concerns/TracksLlmSpans.php new file mode 100644 index 0000000..f6552a1 --- /dev/null +++ b/src/Support/Events/Subscribers/Otel/Concerns/TracksLlmSpans.php @@ -0,0 +1,151 @@ + + */ + private array $llmSpans = []; + + private function registerLlmListeners(InternalEventDispatcher $dispatcher, TracerInterface $tracer): void + { + $dispatcher->listen(ChatModelStart::class, function (ChatModelStart $event) use ($tracer): void { + $model = $event->llm->getModel(); + $provider = $event->llm->getModelProvider()->value; + $toolDefinitions = $event->llm->getToolConfig()?->tools; + + $span = $tracer->spanBuilder('chat ' . $model) + ->setSpanKind(SpanKind::KIND_CLIENT) + ->startSpan(); + + $span->setAttribute(GenAiAttributes::OPERATION_NAME, 'chat'); + $span->setAttribute(GenAiAttributes::SYSTEM, $provider); + $span->setAttribute(GenAiAttributes::PROVIDER_NAME, $provider); + $span->setAttribute(GenAiAttributes::REQUEST_MODEL, $model); + $span->setAttribute(GenAiAttributes::INPUT_MESSAGES, $this->serializeMessages($event->messages)); + + if ($toolDefinitions !== null) { + $span->setAttribute( + GenAiAttributes::TOOL_DEFINITIONS, + (string) json_encode(array_map(fn(Tool $tool): array => $tool->format(), $toolDefinitions)), + ); + } + + if ($event->llm->getMaxTokens() !== null) { + $span->setAttribute(GenAiAttributes::REQUEST_MAX_TOKENS, $event->llm->getMaxTokens()); + } + + if ($event->llm->getTemperature() !== null) { + $span->setAttribute(GenAiAttributes::REQUEST_TEMPERATURE, $event->llm->getTemperature()); + } + + $this->llmSpans[$this->llmContextKey()] = $span; + }); + + $dispatcher->listen(ChatModelEnd::class, function (ChatModelEnd $event): void { + $key = $this->llmContextKey(); + + if (! isset($this->llmSpans[$key])) { + return; + } + + $span = $this->llmSpans[$key]; + + if ($event->result instanceof ChatResult) { + $usage = $event->result->usage; + + $span->setAttribute(GenAiAttributes::USAGE_INPUT_TOKENS, $usage->promptTokens); + + if ($usage->completionTokens !== null) { + $span->setAttribute(GenAiAttributes::USAGE_OUTPUT_TOKENS, $usage->completionTokens); + } + + $responseModel = $event->result->generation->message->metadata?->model; + + if ($responseModel !== null) { + $span->setAttribute(GenAiAttributes::RESPONSE_MODEL, $responseModel); + } + + $span->setAttribute( + GenAiAttributes::OUTPUT_MESSAGES, + $this->serializeMessages(new MessageCollection([$event->result->generation->message])), + ); + } + + $span->end(); + + unset($this->llmSpans[$key]); + }); + + $dispatcher->listen(ChatModelError::class, function (ChatModelError $event): void { + $key = $this->llmContextKey(); + + if (! isset($this->llmSpans[$key])) { + return; + } + + $span = $this->llmSpans[$key]; + + $span->setStatus(StatusCode::STATUS_ERROR, $event->exception->getMessage()); + $span->recordException($event->exception); + $span->end(); + + unset($this->llmSpans[$key]); + }); + + $dispatcher->listen(ChatModelStreamEnd::class, function (ChatModelStreamEnd $event): void { + $key = $this->llmContextKey(); + + if (! isset($this->llmSpans[$key])) { + return; + } + + $span = $this->llmSpans[$key]; + $usage = $event->chunk->usage; + + if ($usage !== null) { + $span->setAttribute(GenAiAttributes::USAGE_INPUT_TOKENS, $usage->promptTokens); + + if ($usage->completionTokens !== null) { + $span->setAttribute(GenAiAttributes::USAGE_OUTPUT_TOKENS, $usage->completionTokens); + } + } + + $span->end(); + + unset($this->llmSpans[$key]); + }); + } + + /** + * LLM events don't carry a run_id, so we use a fixed key per concurrent LLM call. + * Since LLM calls are synchronous within a step, a single key per subscriber is sufficient. + * For concurrent calls, users should provide their own TracerProvider with context propagation. + */ + private function llmContextKey(): string + { + return 'llm_current'; + } +} diff --git a/src/Support/Events/Subscribers/Otel/Concerns/TracksMiddlewareSpans.php b/src/Support/Events/Subscribers/Otel/Concerns/TracksMiddlewareSpans.php new file mode 100644 index 0000000..a29c874 --- /dev/null +++ b/src/Support/Events/Subscribers/Otel/Concerns/TracksMiddlewareSpans.php @@ -0,0 +1,75 @@ + + */ + private array $middlewareSpans = []; + + private function registerMiddlewareListeners(InternalEventDispatcher $dispatcher, TracerInterface $tracer): void + { + $dispatcher->listen(MiddlewareStart::class, function (MiddlewareStart $event) use ($tracer): void { + $middlewareClass = $event->middleware::class; + $shortName = class_basename($middlewareClass); + + $span = $tracer->spanBuilder($event->hook . ' ' . $shortName) + ->setSpanKind(SpanKind::KIND_INTERNAL) + ->startSpan(); + + $span->setAttribute('cortex.middleware.class', $middlewareClass); + $span->setAttribute('cortex.middleware.hook', $event->hook); + + $span->setAttribute('cortex.run_id', $event->config->runId); + + $this->middlewareSpans[$this->middlewareKey($event->middleware::class, $event->hook, $event->config->runId)] = $span; + }); + + $dispatcher->listen(MiddlewareEnd::class, function (MiddlewareEnd $event): void { + $key = $this->middlewareKey($event->middleware::class, $event->hook, $event->config->runId); + + if (! isset($this->middlewareSpans[$key])) { + return; + } + + $this->middlewareSpans[$key]->end(); + + unset($this->middlewareSpans[$key]); + }); + + $dispatcher->listen(MiddlewareError::class, function (MiddlewareError $event): void { + $key = $this->middlewareKey($event->middleware::class, $event->hook, $event->config->runId); + + if (! isset($this->middlewareSpans[$key])) { + return; + } + + $span = $this->middlewareSpans[$key]; + $span->setStatus(StatusCode::STATUS_ERROR, $event->exception->getMessage()); + $span->recordException($event->exception); + $span->end(); + + unset($this->middlewareSpans[$key]); + }); + } + + private function middlewareKey(string $class, string $hook, string $runId): string + { + return $runId . ':' . $class . ':' . $hook; + } +} diff --git a/src/Support/Events/Subscribers/Otel/Concerns/TracksStepSpans.php b/src/Support/Events/Subscribers/Otel/Concerns/TracksStepSpans.php new file mode 100644 index 0000000..96644fb --- /dev/null +++ b/src/Support/Events/Subscribers/Otel/Concerns/TracksStepSpans.php @@ -0,0 +1,79 @@ + + */ + private array $stepSpans = []; + + private function registerStepListeners(InternalEventDispatcher $dispatcher, TracerInterface $tracer): void + { + $dispatcher->listen(AgentStepStart::class, function (AgentStepStart $event) use ($tracer): void { + $runId = $event->config->runId ?? 'default'; + + $span = $tracer->spanBuilder('agent.step') + ->setSpanKind(SpanKind::KIND_INTERNAL) + ->startSpan(); + + $span->setAttribute(GenAiAttributes::AGENT_ID, $event->agent->getId()); + + if ($event->config?->runId !== null) { + $span->setAttribute('cortex.run_id', $event->config->runId); + } + + $this->stepSpans[$runId] = [ + 'span' => $span, + 'scope' => $span->activate(), + ]; + }); + + $dispatcher->listen(AgentStepEnd::class, function (AgentStepEnd $event): void { + $runId = $event->config->runId ?? 'default'; + + if (! isset($this->stepSpans[$runId])) { + return; + } + + ['span' => $span, 'scope' => $scope] = $this->stepSpans[$runId]; + + $scope->detach(); + $span->end(); + + unset($this->stepSpans[$runId]); + }); + + $dispatcher->listen(AgentStepError::class, function (AgentStepError $event): void { + $runId = $event->config->runId ?? 'default'; + + if (! isset($this->stepSpans[$runId])) { + return; + } + + ['span' => $span, 'scope' => $scope] = $this->stepSpans[$runId]; + + $span->setStatus(StatusCode::STATUS_ERROR, $event->exception->getMessage()); + $span->recordException($event->exception); + + $scope->detach(); + $span->end(); + + unset($this->stepSpans[$runId]); + }); + } +} diff --git a/src/Support/Events/Subscribers/Otel/Concerns/TracksToolSpans.php b/src/Support/Events/Subscribers/Otel/Concerns/TracksToolSpans.php new file mode 100644 index 0000000..483b24e --- /dev/null +++ b/src/Support/Events/Subscribers/Otel/Concerns/TracksToolSpans.php @@ -0,0 +1,62 @@ + + */ + private array $toolSpans = []; + + private function registerToolListeners(InternalEventDispatcher $dispatcher, TracerInterface $tracer): void + { + $dispatcher->listen(ToolCallStart::class, function (ToolCallStart $event) use ($tracer): void { + $toolCallId = $event->toolCall->id; + $toolName = $event->toolCall->function->name; + + $span = $tracer->spanBuilder('execute_tool ' . $toolName) + ->setSpanKind(SpanKind::KIND_INTERNAL) + ->startSpan(); + + $span->setAttribute(GenAiAttributes::OPERATION_NAME, 'execute_tool'); + $span->setAttribute(GenAiAttributes::TOOL_NAME, $toolName); + $span->setAttribute(GenAiAttributes::TOOL_CALL_ID, $toolCallId); + $span->setAttribute(GenAiAttributes::TOOL_CALL_ARGUMENTS, $event->toolCall->function->arguments); + + if ($event->config?->runId !== null) { + $span->setAttribute('cortex.run_id', $event->config->runId); + } + + $this->toolSpans[$toolCallId] = $span; + }); + + $dispatcher->listen(ToolCallEnd::class, function (ToolCallEnd $event): void { + $toolCallId = $event->toolMessage->id; + + if (! isset($this->toolSpans[$toolCallId])) { + return; + } + + $span = $this->toolSpans[$toolCallId]; + + $span->setAttribute(GenAiAttributes::TOOL_CALL_RESULT, $event->toolMessage->content); + $span->end(); + + unset($this->toolSpans[$toolCallId]); + }); + } +} diff --git a/src/Support/Events/Subscribers/Otel/GenAiAttributes.php b/src/Support/Events/Subscribers/Otel/GenAiAttributes.php new file mode 100644 index 0000000..28e69a2 --- /dev/null +++ b/src/Support/Events/Subscribers/Otel/GenAiAttributes.php @@ -0,0 +1,48 @@ +tracerProvider->getTracer(self::TRACER_NAME); + + $this->registerAgentListeners($dispatcher, $tracer); + $this->registerStepListeners($dispatcher, $tracer); + $this->registerMiddlewareListeners($dispatcher, $tracer); + $this->registerLlmListeners($dispatcher, $tracer); + $this->registerToolListeners($dispatcher, $tracer); + } +} diff --git a/tests/Unit/Support/Subscribers/OpenTelemetrySubscriberTest.php b/tests/Unit/Support/Subscribers/OpenTelemetrySubscriberTest.php new file mode 100644 index 0000000..ed60378 --- /dev/null +++ b/tests/Unit/Support/Subscribers/OpenTelemetrySubscriberTest.php @@ -0,0 +1,537 @@ +newInstanceWithoutConstructor(); + + $prop = $reflection->getProperty('instance'); + $prop->setValue(null, $instance); + + return $instance; +} + +function makeFakeLlm(): FakeChat +{ + return new FakeChat([ + ChatGeneration::fromMessage(new AssistantMessage('Hello')), + ]); +} + +function makeAgent(): Agent +{ + return new Agent( + id: 'test-agent', + prompt: 'You are a test agent.', + llm: makeFakeLlm(), + ); +} + +function makeConfig(string $runId = 'run-123', string $threadId = 'thread-456'): RuntimeConfig +{ + return new RuntimeConfig(threadId: $threadId, runId: $runId); +} + +describe('OpenTelemetrySubscriber', function (): void { + beforeEach(function (): void { + $this->storage = new ArrayObject(); + $this->tracerProvider = makeTracerProvider($this->storage); + $this->dispatcher = makeDispatcher(); + $this->subscriber = new OpenTelemetrySubscriber($this->tracerProvider); + $this->dispatcher->subscribe($this->subscriber); + }); + + afterEach(function (): void { + $this->tracerProvider->shutdown(); + + $reflection = new ReflectionClass(InternalEventDispatcher::class); + $prop = $reflection->getProperty('instance'); + $prop->setValue(null, null); + }); + + describe('agent spans', function (): void { + test('it creates an invoke_agent span on agent start and end', function (): void { + $agent = makeAgent(); + $config = makeConfig(); + + $this->dispatcher->dispatch(new AgentStart($agent, $config)); + $this->dispatcher->dispatch(new AgentEnd($agent, $config)); + + $spans = $this->storage->getArrayCopy(); + + expect($spans)->toHaveCount(1); + + $span = $spans[0]; + expect($span->getName())->toBe('invoke_agent') + ->and($span->getAttributes()->get('gen_ai.operation.name'))->toBe('invoke_agent') + ->and($span->getAttributes()->get('gen_ai.agent.id'))->toBe('test-agent') + ->and($span->getAttributes()->get('cortex.run_id'))->toBe('run-123') + ->and($span->getAttributes()->get('gen_ai.conversation.id'))->toBe('thread-456') + ->and($span->hasEnded())->toBeTrue(); + }); + + test('it uses agent name in span name when available', function (): void { + $agent = new Agent( + id: 'named-agent', + prompt: 'You are a test agent.', + llm: makeFakeLlm(), + name: 'My Agent', + ); + $config = makeConfig(); + + $this->dispatcher->dispatch(new AgentStart($agent, $config)); + $this->dispatcher->dispatch(new AgentEnd($agent, $config)); + + $spans = $this->storage->getArrayCopy(); + + expect($spans[0]->getName())->toBe('invoke_agent My Agent') + ->and($spans[0]->getAttributes()->get('gen_ai.agent.name'))->toBe('My Agent'); + }); + + test('it creates an invoke_agent span without config', function (): void { + $agent = makeAgent(); + + $this->dispatcher->dispatch(new AgentStart($agent)); + $this->dispatcher->dispatch(new AgentEnd($agent)); + + $spans = $this->storage->getArrayCopy(); + + expect($spans)->toHaveCount(1); + expect($spans[0]->getName())->toBe('invoke_agent') + ->and($spans[0]->getAttributes()->get('cortex.run_id'))->toBeNull() + ->and($spans[0]->getAttributes()->get('gen_ai.conversation.id'))->toBeNull(); + }); + + test('it does not end a span if no matching start was dispatched', function (): void { + $agent = makeAgent(); + $config = makeConfig(); + + $this->dispatcher->dispatch(new AgentEnd($agent, $config)); + + expect($this->storage->getArrayCopy())->toHaveCount(0); + }); + }); + + describe('agent step spans', function (): void { + test('it creates an agent.step span on step start and end', function (): void { + $agent = makeAgent(); + $config = makeConfig(); + + $this->dispatcher->dispatch(new AgentStart($agent, $config)); + $this->dispatcher->dispatch(new AgentStepStart($agent, $config)); + $this->dispatcher->dispatch(new AgentStepEnd($agent, $config)); + $this->dispatcher->dispatch(new AgentEnd($agent, $config)); + + $spans = $this->storage->getArrayCopy(); + + expect($spans)->toHaveCount(2); + + $stepSpan = $spans[0]; + $agentSpan = $spans[1]; + + expect($stepSpan->getName())->toBe('agent.step') + ->and($stepSpan->getAttributes()->get('gen_ai.agent.id'))->toBe('test-agent') + ->and($stepSpan->getAttributes()->get('cortex.run_id'))->toBe('run-123') + ->and($stepSpan->hasEnded())->toBeTrue(); + + expect($agentSpan->getName())->toBe('invoke_agent'); + }); + + test('it records an exception and sets error status on step error', function (): void { + $agent = makeAgent(); + $config = makeConfig(); + $exception = new RuntimeException('Step failed'); + + $this->dispatcher->dispatch(new AgentStart($agent, $config)); + $this->dispatcher->dispatch(new AgentStepStart($agent, $config)); + $this->dispatcher->dispatch(new AgentStepError($agent, $exception, $config)); + $this->dispatcher->dispatch(new AgentEnd($agent, $config)); + + $spans = $this->storage->getArrayCopy(); + + expect($spans)->toHaveCount(2); + + $stepSpan = $spans[0]; + expect($stepSpan->getName())->toBe('agent.step') + ->and($stepSpan->getStatus()->getCode())->toBe(StatusCode::STATUS_ERROR) + ->and($stepSpan->getStatus()->getDescription())->toBe('Step failed') + ->and($stepSpan->getEvents())->toHaveCount(1) + ->and($stepSpan->getEvents()[0]->getName())->toBe('exception'); + }); + }); + + describe('LLM call spans', function (): void { + test('it creates a chat span on LLM start and end', function (): void { + $llm = makeFakeLlm(); + $messages = MessageCollection::make([]); + $result = makeChatResult(100, 50); + + $this->dispatcher->dispatch(new ChatModelStart($llm, $messages)); + $this->dispatcher->dispatch(new ChatModelEnd($llm, $result)); + + $spans = $this->storage->getArrayCopy(); + + expect($spans)->toHaveCount(1); + + $span = $spans[0]; + expect($span->getName())->toBe('chat fake-model') + ->and($span->getAttributes()->get('gen_ai.operation.name'))->toBe('chat') + ->and($span->getAttributes()->get('gen_ai.system'))->toBe('openai') + ->and($span->getAttributes()->get('gen_ai.provider.name'))->toBe('openai') + ->and($span->getAttributes()->get('gen_ai.request.model'))->toBe('fake-model') + ->and($span->getAttributes()->get('gen_ai.usage.input_tokens'))->toBe(100) + ->and($span->getAttributes()->get('gen_ai.usage.output_tokens'))->toBe(50) + ->and($span->hasEnded())->toBeTrue(); + }); + + test('it records exception and sets error status on LLM error', function (): void { + $llm = makeFakeLlm(); + $messages = MessageCollection::make([]); + $exception = new RuntimeException('API timeout'); + + $this->dispatcher->dispatch(new ChatModelStart($llm, $messages)); + $this->dispatcher->dispatch(new ChatModelError($llm, [], $exception)); + + $spans = $this->storage->getArrayCopy(); + + expect($spans)->toHaveCount(1); + + $span = $spans[0]; + expect($span->getName())->toBe('chat fake-model') + ->and($span->getStatus()->getCode())->toBe(StatusCode::STATUS_ERROR) + ->and($span->getStatus()->getDescription())->toBe('API timeout') + ->and($span->getEvents())->toHaveCount(1) + ->and($span->getEvents()[0]->getName())->toBe('exception'); + }); + + test('it does not set output tokens attribute when completion tokens is null', function (): void { + $llm = makeFakeLlm(); + $messages = MessageCollection::make([]); + $result = makeChatResult(100, null); + + $this->dispatcher->dispatch(new ChatModelStart($llm, $messages)); + $this->dispatcher->dispatch(new ChatModelEnd($llm, $result)); + + $spans = $this->storage->getArrayCopy(); + $span = $spans[0]; + + expect($span->getAttributes()->get('gen_ai.usage.input_tokens'))->toBe(100) + ->and($span->getAttributes()->get('gen_ai.usage.output_tokens'))->toBeNull(); + }); + + test('it closes the chat_model.call span on stream end with usage', function (): void { + $llm = makeFakeLlm(); + $messages = MessageCollection::make([]); + $usage = new Usage(promptTokens: 80, completionTokens: 40); + $chunk = new ChatGenerationChunk(type: ChunkType::ChatModelEnd, usage: $usage, isFinal: true); + + $this->dispatcher->dispatch(new ChatModelStart($llm, $messages)); + $this->dispatcher->dispatch(new ChatModelStreamEnd($llm, $chunk)); + + $spans = $this->storage->getArrayCopy(); + + expect($spans)->toHaveCount(1); + + $span = $spans[0]; + expect($span->getName())->toBe('chat fake-model') + ->and($span->getAttributes()->get('gen_ai.usage.input_tokens'))->toBe(80) + ->and($span->getAttributes()->get('gen_ai.usage.output_tokens'))->toBe(40) + ->and($span->hasEnded())->toBeTrue(); + }); + + test('it closes the chat span on stream end without usage', function (): void { + $llm = makeFakeLlm(); + $messages = MessageCollection::make([]); + $chunk = new ChatGenerationChunk(type: ChunkType::ChatModelEnd, isFinal: true); + + $this->dispatcher->dispatch(new ChatModelStart($llm, $messages)); + $this->dispatcher->dispatch(new ChatModelStreamEnd($llm, $chunk)); + + $spans = $this->storage->getArrayCopy(); + + expect($spans)->toHaveCount(1); + expect($spans[0]->hasEnded())->toBeTrue() + ->and($spans[0]->getAttributes()->get('gen_ai.usage.input_tokens'))->toBeNull(); + }); + }); + + describe('tool call spans', function (): void { + test('it creates an execute_tool span on tool start and end', function (): void { + $config = makeConfig(); + $toolCall = new ToolCall( + id: 'call-abc123', + function: new FunctionCall('search', [ + 'query' => 'test', + ]), + ); + $toolMessage = new ToolMessage('result', 'call-abc123', 'search'); + + $this->dispatcher->dispatch(new ToolCallStart($toolCall, $config)); + $this->dispatcher->dispatch(new ToolCallEnd($toolMessage, $config)); + + $spans = $this->storage->getArrayCopy(); + + expect($spans)->toHaveCount(1); + + $span = $spans[0]; + + expect($span->getName())->toBe('execute_tool search') + ->and($span->getAttributes()->get('gen_ai.operation.name'))->toBe('execute_tool') + ->and($span->getAttributes()->get('gen_ai.tool.name'))->toBe('search') + ->and($span->getAttributes()->get('gen_ai.tool.call.id'))->toBe('call-abc123') + ->and($span->getAttributes()->get('cortex.run_id'))->toBe('run-123') + ->and($span->hasEnded())->toBeTrue(); + }); + + test('it handles multiple concurrent tool calls independently', function (): void { + $config = makeConfig(); + $toolCall1 = new ToolCall(id: 'call-1', function: new FunctionCall('search', [])); + $toolCall2 = new ToolCall(id: 'call-2', function: new FunctionCall('calculator', [])); + $toolMessage1 = new ToolMessage('result1', 'call-1'); + $toolMessage2 = new ToolMessage('result2', 'call-2'); + + $this->dispatcher->dispatch(new ToolCallStart($toolCall1, $config)); + $this->dispatcher->dispatch(new ToolCallStart($toolCall2, $config)); + $this->dispatcher->dispatch(new ToolCallEnd($toolMessage1, $config)); + $this->dispatcher->dispatch(new ToolCallEnd($toolMessage2, $config)); + + $spans = $this->storage->getArrayCopy(); + + expect($spans)->toHaveCount(2); + + $names = array_map(fn($s) => $s->getAttributes()->get('gen_ai.tool.name'), $spans); + expect($names)->toContain('search') + ->toContain('calculator'); + }); + + test('it does not end a tool span if no matching start was dispatched', function (): void { + $toolMessage = new ToolMessage('result', 'nonexistent-id'); + + $this->dispatcher->dispatch(new ToolCallEnd($toolMessage)); + + expect($this->storage->getArrayCopy())->toHaveCount(0); + }); + }); + + describe('span hierarchy', function (): void { + test('step span is a child of agent span', function (): void { + $agent = makeAgent(); + $config = makeConfig(); + + $this->dispatcher->dispatch(new AgentStart($agent, $config)); + $this->dispatcher->dispatch(new AgentStepStart($agent, $config)); + $this->dispatcher->dispatch(new AgentStepEnd($agent, $config)); + $this->dispatcher->dispatch(new AgentEnd($agent, $config)); + + $spans = $this->storage->getArrayCopy(); + expect($spans)->toHaveCount(2); + + $stepSpan = $spans[0]; + $agentSpan = $spans[1]; + + expect($stepSpan->getParentSpanId())->toBe($agentSpan->getSpanId()); + }); + + test('LLM span is a child of step span', function (): void { + $agent = makeAgent(); + $config = makeConfig(); + $llm = makeFakeLlm(); + $messages = MessageCollection::make([]); + $result = makeChatResult(10, 5); + + $this->dispatcher->dispatch(new AgentStart($agent, $config)); + $this->dispatcher->dispatch(new AgentStepStart($agent, $config)); + $this->dispatcher->dispatch(new ChatModelStart($llm, $messages)); + $this->dispatcher->dispatch(new ChatModelEnd($llm, $result)); + $this->dispatcher->dispatch(new AgentStepEnd($agent, $config)); + $this->dispatcher->dispatch(new AgentEnd($agent, $config)); + + $spans = $this->storage->getArrayCopy(); + expect($spans)->toHaveCount(3); + + $llmSpan = $spans[0]; + $stepSpan = $spans[1]; + $agentSpan = $spans[2]; + + expect($llmSpan->getName())->toBe('chat fake-model') + ->and($stepSpan->getName())->toBe('agent.step') + ->and($agentSpan->getName())->toBe('invoke_agent'); + + expect($llmSpan->getParentSpanId())->toBe($stepSpan->getSpanId()) + ->and($stepSpan->getParentSpanId())->toBe($agentSpan->getSpanId()); + }); + + test('tool span is a child of step span', function (): void { + $agent = makeAgent(); + $config = makeConfig(); + $toolCall = new ToolCall(id: 'call-1', function: new FunctionCall('search', [])); + $toolMessage = new ToolMessage('result', 'call-1'); + + $this->dispatcher->dispatch(new AgentStart($agent, $config)); + $this->dispatcher->dispatch(new AgentStepStart($agent, $config)); + $this->dispatcher->dispatch(new ToolCallStart($toolCall, $config)); + $this->dispatcher->dispatch(new ToolCallEnd($toolMessage, $config)); + $this->dispatcher->dispatch(new AgentStepEnd($agent, $config)); + $this->dispatcher->dispatch(new AgentEnd($agent, $config)); + + $spans = $this->storage->getArrayCopy(); + expect($spans)->toHaveCount(3); + + $toolSpan = $spans[0]; + $stepSpan = $spans[1]; + + expect($toolSpan->getName())->toBe('execute_tool search') + ->and($toolSpan->getParentSpanId())->toBe($stepSpan->getSpanId()); + }); + }); + + describe('middleware spans', function (): void { + test('it creates a middleware span on start and end', function (): void { + $config = makeConfig(); + $middleware = new class () extends AbstractMiddleware implements BeforePromptMiddleware {}; + + $this->dispatcher->dispatch(new MiddlewareStart($middleware, $config, 'beforePrompt')); + $this->dispatcher->dispatch(new MiddlewareEnd($middleware, $config, 'beforePrompt')); + + $spans = $this->storage->getArrayCopy(); + + expect($spans)->toHaveCount(1); + + $span = $spans[0]; + expect($span->getName())->toStartWith('beforePrompt ') + ->and($span->getAttributes()->get('cortex.middleware.class'))->toBe($middleware::class) + ->and($span->getAttributes()->get('cortex.middleware.hook'))->toBe('beforePrompt') + ->and($span->getAttributes()->get('cortex.run_id'))->toBe('run-123') + ->and($span->hasEnded())->toBeTrue(); + }); + + test('it records exception and sets error status on middleware error', function (): void { + $config = makeConfig(); + $middleware = new class () extends AbstractMiddleware implements BeforePromptMiddleware {}; + $exception = new RuntimeException('Middleware failed'); + + $this->dispatcher->dispatch(new MiddlewareStart($middleware, $config, 'beforePrompt')); + $this->dispatcher->dispatch(new MiddlewareError($middleware, $config, 'beforePrompt', $exception)); + + $spans = $this->storage->getArrayCopy(); + + expect($spans)->toHaveCount(1); + + $span = $spans[0]; + expect($span->getStatus()->getCode())->toBe(StatusCode::STATUS_ERROR) + ->and($span->getStatus()->getDescription())->toBe('Middleware failed') + ->and($span->getEvents())->toHaveCount(1) + ->and($span->getEvents()[0]->getName())->toBe('exception'); + }); + + test('middleware span is a child of step span', function (): void { + $agent = makeAgent(); + $config = makeConfig(); + $middleware = new class () extends AbstractMiddleware implements BeforePromptMiddleware {}; + + $this->dispatcher->dispatch(new AgentStart($agent, $config)); + $this->dispatcher->dispatch(new AgentStepStart($agent, $config)); + $this->dispatcher->dispatch(new MiddlewareStart($middleware, $config, 'beforePrompt')); + $this->dispatcher->dispatch(new MiddlewareEnd($middleware, $config, 'beforePrompt')); + $this->dispatcher->dispatch(new AgentStepEnd($agent, $config)); + $this->dispatcher->dispatch(new AgentEnd($agent, $config)); + + $spans = $this->storage->getArrayCopy(); + expect($spans)->toHaveCount(3); + + $middlewareSpan = $spans[0]; + $stepSpan = $spans[1]; + + expect($middlewareSpan->getParentSpanId())->toBe($stepSpan->getSpanId()); + }); + + test('it does not end a span if no matching start was dispatched', function (): void { + $config = makeConfig(); + $middleware = new class () extends AbstractMiddleware implements BeforePromptMiddleware {}; + + $this->dispatcher->dispatch(new MiddlewareEnd($middleware, $config, 'beforePrompt')); + + expect($this->storage->getArrayCopy())->toHaveCount(0); + }); + }); + + describe('multiple sequential runs', function (): void { + test('it tracks spans independently per run_id for sequential runs', function (): void { + $agent = makeAgent(); + $config1 = makeConfig('run-1', 'thread-1'); + $config2 = makeConfig('run-2', 'thread-2'); + + $this->dispatcher->dispatch(new AgentStart($agent, $config1)); + $this->dispatcher->dispatch(new AgentEnd($agent, $config1)); + + $this->dispatcher->dispatch(new AgentStart($agent, $config2)); + $this->dispatcher->dispatch(new AgentEnd($agent, $config2)); + + $spans = $this->storage->getArrayCopy(); + + expect($spans)->toHaveCount(2); + + $runIds = array_map(fn($s) => $s->getAttributes()->get('cortex.run_id'), $spans); + expect($runIds)->toContain('run-1') + ->toContain('run-2'); + }); + }); +}); diff --git a/workbench/app/Providers/CortexServiceProvider.php b/workbench/app/Providers/CortexServiceProvider.php index 7450636..c1a726c 100644 --- a/workbench/app/Providers/CortexServiceProvider.php +++ b/workbench/app/Providers/CortexServiceProvider.php @@ -132,8 +132,9 @@ public function boot(): void name: 'Generic Assistant', description: 'A helpful assistant that can answer questions.', prompt: 'You are a helpful assistant.', - llm: 'lmstudio/qwen3.5-9b-mlx', - // llm: 'ollama/gpt-oss:20b', + // llm: 'lmstudio/qwen3.5-9b-mlx', + // llm: 'ollama/qwen3.5:9b', + llm: 'lmstudio/openai/gpt-oss-20b', tools: [ // $translationAgent->asTool('translate', 'Translate text from one language to another.'), // $storyIdeaGenerator->asTool('generate_story_idea', 'Generate a story idea about a given topic.'),