feat: Otel Observabillity (#18)

* feat: Otel Observabillity

* 🎨

* refactor

* install semconv

* fix

* fix

* fix

* add middleware

* cleanup

* tweaks

Author: tymondesigns

composer.json

@@ -23,6 +23,10 @@
         "guzzlehttp/psr7": "^2.8",
         "illuminate/collections": "^12.49",
         "laravel/prompts": "^0.3.8",
+        "open-telemetry/api": "^1.8",
+        "open-telemetry/exporter-otlp": "^1.4",
+        "open-telemetry/sdk": "^1.13",
+        "open-telemetry/sem-conv": "^1.38",
         "openai-php/client": "^0.18",
         "php-mcp/client": "^1.0",
         "psr-discovery/cache-implementations": "^1.2",
@@ -104,7 +108,8 @@
         "allow-plugins": {
             "dealerdirect/phpcodesniffer-composer-installer": true,
             "pestphp/pest-plugin": true,
-            "php-http/discovery": true
+            "php-http/discovery": true,
+            "tbachert/spi": true
         }
     },
     "extra": {

config/cortex.php

@@ -5,6 +5,7 @@
 use Cortex\LLM\Enums\LLMDriver;
 use Cortex\LLM\Enums\StreamingProtocol;
 use Cortex\Agents\Prebuilt\WeatherAgent;
+use OpenTelemetry\Contrib\Otlp\Protocols;
 use Cortex\ModelInfo\Providers\OllamaModelInfoProvider;
 use Cortex\ModelInfo\Providers\LiteLLMModelInfoProvider;
 use Cortex\ModelInfo\Providers\LMStudioModelInfoProvider;
@@ -367,4 +368,45 @@
     |
     */
     'default_streaming_protocol' => StreamingProtocol::Vercel,
+
+    /*
+    |--------------------------------------------------------------------------
+    | OpenTelemetry Tracing
+    |--------------------------------------------------------------------------
+    |
+    | Configure OpenTelemetry tracing to export spans for agent runs, LLM calls,
+    | tool executions, and agent steps to any OTLP-compatible backend
+    | (e.g. Jaeger, Grafana Tempo, Honeycomb, Datadog).
+    |
+    | Set CORTEX_TRACING_ENABLED=true and point OTEL_EXPORTER_OTLP_ENDPOINT
+    | at your collector or backend to get started.
+    |
+    */
+    'tracing' => [
+        'enabled' => env('CORTEX_TRACING_ENABLED', false),
+
+        'exporter' => [
+            /**
+             * The full OTLP traces endpoint URL.
+             * For HTTP/protobuf (default): http://localhost:4318/v1/traces
+             * For gRPC: http://localhost:4317
+             */
+            'endpoint' => env('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://localhost:4318/v1/traces'),
+
+            /**
+             * Supported: "http/protobuf", "http/json"
+             */
+            'protocol' => env('OTEL_EXPORTER_OTLP_PROTOCOL', Protocols::HTTP_PROTOBUF),
+
+            /**
+             * The headers to send with the request. Comma separated list of key=value pairs.
+             */
+            'headers' => env('OTEL_EXPORTER_OTLP_HEADERS', ''),
+        ],
+
+        /**
+         * The service name reported to the tracing backend.
+         */
+        'service_name' => env('OTEL_SERVICE_NAME', 'cortex'),
+    ],
 ];

src/Agents/Middleware/AfterModelWrapper.php

@@ -5,8 +5,14 @@
 namespace Cortex\Agents\Middleware;
 
 use Closure;
+use Throwable;
+use Cortex\Events\MiddlewareEnd;
+use Cortex\Events\MiddlewareError;
+use Cortex\Events\MiddlewareStart;
 use Cortex\Pipeline\RuntimeConfig;
 use Cortex\Support\Traits\CanPipe;
+use Cortex\Support\Traits\DispatchesEvents;
+use Cortex\Events\Contracts\MiddlewareEvent;
 use Cortex\Agents\Contracts\AfterModelMiddleware;
 
 /**
@@ -16,18 +22,36 @@
 class AfterModelWrapper implements AfterModelMiddleware
 {
     use CanPipe;
+    use DispatchesEvents;
 
     public function __construct(
         protected AfterModelMiddleware $middleware,
     ) {}
 
     public function handlePipeable(mixed $payload, RuntimeConfig $config, Closure $next): mixed
     {
-        return $this->middleware->afterModel($payload, $config, $next);
+        $this->dispatchEvent(new MiddlewareStart($this->middleware, $config, 'afterModel'));
+
+        try {
+            $result = $this->middleware->afterModel($payload, $config, $next);
+        } catch (Throwable $e) {
+            $this->dispatchEvent(new MiddlewareError($this->middleware, $config, 'afterModel', $e));
+
+            throw $e;
+        }
+
+        $this->dispatchEvent(new MiddlewareEnd($this->middleware, $config, 'afterModel'));
+
+        return $result;
     }
 
     public function afterModel(mixed $payload, RuntimeConfig $config, Closure $next): mixed
     {
         return $this->handlePipeable($payload, $config, $next);
     }
+
+    protected function eventBelongsToThisInstance(object $event): bool
+    {
+        return $event instanceof MiddlewareEvent && $event->middleware === $this->middleware;
+    }
 }

src/Agents/Middleware/BeforeModelWrapper.php

@@ -5,8 +5,14 @@
 namespace Cortex\Agents\Middleware;
 
 use Closure;
+use Throwable;
+use Cortex\Events\MiddlewareEnd;
+use Cortex\Events\MiddlewareError;
+use Cortex\Events\MiddlewareStart;
 use Cortex\Pipeline\RuntimeConfig;
 use Cortex\Support\Traits\CanPipe;
+use Cortex\Support\Traits\DispatchesEvents;
+use Cortex\Events\Contracts\MiddlewareEvent;
 use Cortex\Agents\Contracts\BeforeModelMiddleware;
 
 /**
@@ -16,18 +22,36 @@
 class BeforeModelWrapper implements BeforeModelMiddleware
 {
     use CanPipe;
+    use DispatchesEvents;
 
     public function __construct(
         protected BeforeModelMiddleware $middleware,
     ) {}
 
     public function handlePipeable(mixed $payload, RuntimeConfig $config, Closure $next): mixed
     {
-        return $this->middleware->beforeModel($payload, $config, $next);
+        $this->dispatchEvent(new MiddlewareStart($this->middleware, $config, 'beforeModel'));
+
+        try {
+            $result = $this->middleware->beforeModel($payload, $config, $next);
+        } catch (Throwable $e) {
+            $this->dispatchEvent(new MiddlewareError($this->middleware, $config, 'beforeModel', $e));
+
+            throw $e;
+        }
+
+        $this->dispatchEvent(new MiddlewareEnd($this->middleware, $config, 'beforeModel'));
+
+        return $result;
     }
 
     public function beforeModel(mixed $payload, RuntimeConfig $config, Closure $next): mixed
     {
         return $this->handlePipeable($payload, $config, $next);
     }
+
+    protected function eventBelongsToThisInstance(object $event): bool
+    {
+        return $event instanceof MiddlewareEvent && $event->middleware === $this->middleware;
+    }
 }

src/Agents/Middleware/BeforePromptWrapper.php

@@ -5,8 +5,14 @@
 namespace Cortex\Agents\Middleware;
 
 use Closure;
+use Throwable;
+use Cortex\Events\MiddlewareEnd;
+use Cortex\Events\MiddlewareError;
+use Cortex\Events\MiddlewareStart;
 use Cortex\Pipeline\RuntimeConfig;
 use Cortex\Support\Traits\CanPipe;
+use Cortex\Support\Traits\DispatchesEvents;
+use Cortex\Events\Contracts\MiddlewareEvent;
 use Cortex\Agents\Contracts\BeforePromptMiddleware;
 
 /**
@@ -16,18 +22,36 @@
 class BeforePromptWrapper implements BeforePromptMiddleware
 {
     use CanPipe;
+    use DispatchesEvents;
 
     public function __construct(
         protected BeforePromptMiddleware $middleware,
     ) {}
 
     public function handlePipeable(mixed $payload, RuntimeConfig $config, Closure $next): mixed
     {
-        return $this->middleware->beforePrompt($payload, $config, $next);
+        $this->dispatchEvent(new MiddlewareStart($this->middleware, $config, 'beforePrompt'));
+
+        try {
+            $result = $this->middleware->beforePrompt($payload, $config, $next);
+        } catch (Throwable $e) {
+            $this->dispatchEvent(new MiddlewareError($this->middleware, $config, 'beforePrompt', $e));
+
+            throw $e;
+        }
+
+        $this->dispatchEvent(new MiddlewareEnd($this->middleware, $config, 'beforePrompt'));
+
+        return $result;
     }
 
     public function beforePrompt(mixed $payload, RuntimeConfig $config, Closure $next): mixed
     {
         return $this->handlePipeable($payload, $config, $next);
     }
+
+    protected function eventBelongsToThisInstance(object $event): bool
+    {
+        return $event instanceof MiddlewareEvent && $event->middleware === $this->middleware;
+    }
 }

src/Agents/Prebuilt/WeatherAgent.php

@@ -55,7 +55,8 @@ public function prompt(): ChatPromptTemplate|ChatPromptBuilder|string
 
     public function llm(): LLM|string|null
     {
-        return Cortex::llm('ollama', 'qwen3.5:9b')->ignoreFeatures();
+        return Cortex::llm('lmstudio/openai/gpt-oss-20b')->ignoreFeatures();
+        // return Cortex::llm('ollama', 'qwen3.5:9b')->ignoreFeatures();
         // return Cortex::llm('lmstudio/qwen3.5-9b-mlx')->ignoreFeatures();
         // return Cortex::llm('anthropic', 'claude-haiku-4-5')->ignoreFeatures();
         // return Cortex::llm('openai', 'gpt-5-mini')->ignoreFeatures();

src/Agents/Registry.php

@@ -29,7 +29,6 @@ public function register(Agent|string $agent, ?string $idOverride = null): void
                 );
             }
 
-            // @phpstan-ignore function.alreadyNarrowedType
             if (! is_subclass_of($agent, AbstractAgentBuilder::class)) {
                 throw new InvalidArgumentException(
                     sprintf(

src/CortexServiceProvider.php

@@ -5,25 +5,31 @@
 namespace Cortex;
 
 use Throwable;
-use Monolog\Logger;
 use Cortex\LLM\LLMManager;
 use Cortex\Agents\Registry;
 use Cortex\Console\AgentChat;
 use Cortex\LLM\Contracts\LLM;
 use Cortex\Mcp\McpServerManager;
-use Monolog\Handler\StreamHandler;
-use Monolog\Formatter\LineFormatter;
 use Illuminate\Support\Facades\Blade;
 use Cortex\ModelInfo\ModelInfoFactory;
 use Spatie\LaravelPackageTools\Package;
 use Cortex\Embeddings\EmbeddingsManager;
 use Cortex\Prompts\PromptFactoryManager;
+use OpenTelemetry\Contrib\Otlp\Protocols;
 use Cortex\Embeddings\Contracts\Embeddings;
 use Cortex\Prompts\Contracts\PromptFactory;
+use OpenTelemetry\Contrib\Otlp\SpanExporter;
+use OpenTelemetry\SDK\Resource\ResourceInfo;
 use Illuminate\Contracts\Container\Container;
 use Cortex\Support\Events\InternalEventDispatcher;
+use OpenTelemetry\SDK\Common\Attribute\Attributes;
+use OpenTelemetry\SDK\Common\Util\ShutdownHandler;
+use OpenTelemetry\SDK\Trace\TracerProviderBuilder;
+use OpenTelemetry\SDK\Resource\ResourceInfoFactory;
 use Spatie\LaravelPackageTools\PackageServiceProvider;
-use Cortex\Support\Events\Subscribers\LoggingSubscriber;
+use OpenTelemetry\Contrib\Otlp\OtlpHttpTransportFactory;
+use OpenTelemetry\SDK\Trace\SpanProcessor\SimpleSpanProcessor;
+use Cortex\Support\Events\Subscribers\Otel\OpenTelemetrySubscriber;
 
 class CortexServiceProvider extends PackageServiceProvider
 {
@@ -55,6 +61,7 @@ public function packageBooted(): void
         $this->registerBladeDirectives();
 
         $this->setupLogging();
+        $this->setupTracing();
     }
 
     protected function registerBladeDirectives(): void
@@ -163,12 +170,67 @@ protected function setupLogging(): void
         }
 
         // TODO: This will be configurable.
-        $logger = new Logger('cortex');
-        $handler = new StreamHandler('php://stdout');
-        $handler->setFormatter(new LineFormatter());
+        // $logger = new Logger('cortex');
+        // $handler = new StreamHandler('php://stdout');
+        // $handler->setFormatter(new LineFormatter());
 
-        $logger->pushHandler($handler);
+        // $logger->pushHandler($handler);
 
-        InternalEventDispatcher::instance()->subscribe(new LoggingSubscriber($logger));
+        // InternalEventDispatcher::instance()->subscribe(new LoggingSubscriber($logger));
+    }
+
+    protected function setupTracing(): void
+    {
+        if ($this->app->runningUnitTests()) {
+            return;
+        }
+
+        if (! config('cortex.tracing.enabled', false)) {
+            return;
+        }
+
+        $endpoint = (string) config('cortex.tracing.exporter.endpoint', 'http://localhost:4318/v1/traces');
+        $protocol = (string) config('cortex.tracing.exporter.protocol', Protocols::HTTP_PROTOBUF);
+        $serviceName = (string) config('cortex.tracing.service_name', 'cortex');
+        $rawHeaders = config('cortex.tracing.exporter.headers', '');
+        $headers = [];
+
+        foreach (explode(',', $rawHeaders) as $header) {
+            $header = trim($header);
+
+            if ($header === '') {
+                continue;
+            }
+
+            $parts = explode('=', $header, 2);
+
+            if (count($parts) === 2) {
+                [$key, $value] = $parts;
+                $headers[trim($key)] = trim($value, " \t\n\r\0\x0B\"'");
+            }
+        }
+
+        $resource = ResourceInfoFactory::emptyResource()->merge(
+            ResourceInfo::create(Attributes::create([
+                'service.name' => $serviceName,
+            ])),
+        );
+
+        $transport = new OtlpHttpTransportFactory()->create(
+            $endpoint,
+            Protocols::contentType($protocol),
+            $headers,
+        );
+
+        $exporter = new SpanExporter($transport);
+
+        $tracerProvider = new TracerProviderBuilder()
+            ->addSpanProcessor(new SimpleSpanProcessor($exporter))
+            ->setResource($resource)
+            ->build();
+
+        ShutdownHandler::register($tracerProvider->shutdown(...));
+
+        InternalEventDispatcher::instance()->subscribe(new OpenTelemetrySubscriber($tracerProvider));
     }
 }

src/Events/Contracts/MiddlewareEvent.php

@@ -0,0 +1,15 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Cortex\Events\Contracts;
+
+use Cortex\Pipeline\RuntimeConfig;
+use Cortex\Agents\Contracts\Middleware;
+
+interface MiddlewareEvent extends CortexEvent
+{
+    public Middleware $middleware { get; }
+
+    public RuntimeConfig $config { get; }
+}