Skip to content

Commit 6067b83

Browse files
tymondesignstymondesigns
committed
add caching support
1 parent 9713edc commit 6067b83

File tree

13 files changed

+153
-46
lines changed

13 files changed

+153
-46
lines changed

‎composer.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
"psr-discovery/cache-implementations": "^1.2",
2828
"psr-discovery/event-dispatcher-implementations": "^1.1",
2929
"react/async": "^4.3",
30+
"saloonphp/cache-plugin": "^3.0",
3031
"saloonphp/saloon": "^3.14",
3132
"spatie/laravel-package-tools": "^1.17"
3233
},

‎config/cortex.php‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,12 @@
2323
'llm' => [
2424
'default' => env('CORTEX_DEFAULT_LLM', 'openai'),
2525

26+
'cache' => [
27+
'enabled' => env('CORTEX_LLM_CACHE_ENABLED', false),
28+
'store' => env('CORTEX_LLM_CACHE_STORE'),
29+
'ttl' => env('CORTEX_LLM_CACHE_TTL', 3600),
30+
],
31+
2632
'openai' => [
2733
'driver' => LLMDriver::OpenAIChat,
2834
'options' => [

‎src/Agents/Prebuilt/WeatherAgent.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ public function llm(): LLM|string|null
4646
{
4747
// return Cortex::llm('ollama', 'gpt-oss:20b')->ignoreFeatures();
4848
// return Cortex::llm('openai', 'gpt-4.1-mini')->ignoreFeatures();
49-
// return Cortex::llm('lmstudio', 'openai/gpt-oss-20b')->ignoreFeatures();
50-
return Cortex::llm('anthropic', 'claude-sonnet-4-5-20250929')->ignoreFeatures();
49+
return Cortex::llm('lmstudio', 'openai/gpt-oss-20b')->ignoreFeatures();
50+
// return Cortex::llm('anthropic', 'claude-sonnet-4-5-20250929')->ignoreFeatures();
5151
}
5252

5353
#[Override]

‎src/Http/Controllers/AgentsController.php‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,13 @@ public function invoke(string $agent, Request $request): JsonResponse
4646
// dump($event->exception->getMessage());
4747
// dump($event->exception->getTraceAsString());
4848
});
49-
$result = $agent->invoke(input: $request->all());
49+
50+
$result = $agent->invoke(
51+
messages: $request->has('message') ? [
52+
new UserMessage($request->input('message')),
53+
] : [],
54+
input: $request->all(),
55+
);
5056
} catch (Throwable $e) {
5157
return response()->json([
5258
'error' => $e->getMessage(),

‎src/LLM/Data/ChatResult.php‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,13 @@
1515

1616
/**
1717
* @param array<int, mixed>|null $rawResponse
18+
* @param array<string, mixed> $metadata
1819
*/
1920
public function __construct(
2021
public ChatGeneration $generation,
2122
public Usage $usage,
2223
public ?array $rawResponse = null,
24+
public array $metadata = [],
2325
) {
2426
$this->parsedOutput = $this->generation->parsedOutput;
2527
}
@@ -50,6 +52,7 @@ public function toArray(): array
5052
'generation' => $this->generation,
5153
'usage' => $this->usage,
5254
'raw_response' => $this->rawResponse,
55+
'metadata' => $this->metadata,
5356
];
5457
}
5558
}

‎src/LLM/Drivers/Anthropic/AnthropicChat.php‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,21 @@ public function invoke(
7474

7575
$this->dispatchEvent(new ChatModelStart($this, $messages, $params));
7676

77+
if ($this->streaming) {
78+
$params['stream'] = true;
79+
}
80+
7781
try {
82+
$response = $this->client->messages()->create(
83+
parameters: $params,
84+
cacheEnabled: $this->useCache,
85+
);
86+
87+
$isCached = $response->isCached();
88+
7889
return $this->streaming
79-
? $this->mapStreamResponse($this->client->messages()->stream($params))
80-
: $this->mapResponse($this->client->messages()->create($params));
90+
? $this->mapStreamResponse($response->dtoOrFail(), $isCached)
91+
: $this->mapResponse($response->dtoOrFail(), $isCached);
8192
} catch (Throwable $e) {
8293
$this->dispatchEvent(new ChatModelError($this, $params, $e));
8394

‎src/LLM/Drivers/Anthropic/Concerns/MapStreamResponse.php‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,9 @@ trait MapStreamResponse
7373
*
7474
* @param \Cortex\SDK\Anthropic\Data\Messages\MessageStream<\Cortex\SDK\Anthropic\Contracts\StreamEvent> $response
7575
*/
76-
protected function mapStreamResponse(MessageStream $response): ChatStreamResult
76+
protected function mapStreamResponse(MessageStream $response, bool $isCached = false): ChatStreamResult
7777
{
78-
return new ChatStreamResult(function () use ($response): Generator {
78+
return new ChatStreamResult(function () use ($response, $isCached): Generator {
7979
$this->resetStreamState();
8080

8181
yield from $this->streamBuffer?->drain() ?? [];
@@ -96,7 +96,7 @@ protected function mapStreamResponse(MessageStream $response): ChatStreamResult
9696

9797
$this->processStreamEvent($event);
9898

99-
$chatGenerationChunk = $this->buildChunk($event, $chunkType, $messageId);
99+
$chatGenerationChunk = $this->buildChunk($event, $chunkType, $messageId, $isCached);
100100
$chatGenerationChunk = $this->applyOutputParserIfApplicable($chatGenerationChunk);
101101

102102
$this->dispatchEvent(new ChatModelStream($this, $chatGenerationChunk));
@@ -292,6 +292,7 @@ private function buildChunk(
292292
StreamEvent $event,
293293
ChunkType $chunkType,
294294
?string $messageId,
295+
bool $isCached = false,
295296
): ChatGenerationChunk {
296297
$finishReason = $event instanceof MessageDelta
297298
? $this->mapFinishReason($event->stopReason)
@@ -325,6 +326,9 @@ private function buildChunk(
325326
contentSoFar: $this->buildContentSnapshot(),
326327
isFinal: $usage !== null,
327328
rawChunk: $this->includeRaw ? $event->raw() : null,
329+
metadata: [
330+
'is_cached' => $isCached,
331+
],
328332
);
329333
}
330334

‎src/LLM/Drivers/Anthropic/Concerns/MapsResponse.php‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ trait MapsResponse
2525
/**
2626
* Map a standard (non-streaming) response to a ChatResult.
2727
*/
28-
protected function mapResponse(MessageResponse $message): ChatResult
28+
protected function mapResponse(MessageResponse $message, bool $isCached = false): ChatResult
2929
{
3030
$usage = $this->mapUsage($message->usage);
3131
$finishReason = $this->mapFinishReason($message->stopReason);
@@ -78,6 +78,9 @@ protected function mapResponse(MessageResponse $message): ChatResult
7878
$generation,
7979
$usage,
8080
$rawResponse, // @phpstan-ignore argument.type
81+
metadata: [
82+
'is_cached' => $isCached,
83+
],
8184
);
8285
}
8386

‎src/LLM/LLMManager.php‎

Lines changed: 15 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,10 @@ protected function createDriver($driver): LLM // @pest-ignore-type
7171
? $driver->value
7272
: $driver;
7373

74+
if ($driver === 'cache') {
75+
throw new InvalidArgumentException('Invalid driver - cache keyword is reserved for internal use.');
76+
}
77+
7478
if (isset($this->customCreators[$driver])) {
7579
return $this->callCustomCreator($config);
7680
}
@@ -131,26 +135,17 @@ public function createOpenAIResponsesDriver(array $config, string $name): OpenAI
131135
*/
132136
public function createAnthropicDriver(array $config, string $name): AnthropicChat
133137
{
134-
// $client = Anthropic::factory()
135-
// ->withApiKey(Arr::get($config, 'options.api_key') ?? '')
136-
// ->withHttpHeader('anthropic-version', Arr::get($config, 'options.version', '2023-06-01'));
137-
138-
// foreach (Arr::get($config, 'options.headers', []) as $key => $value) {
139-
// $client->withHttpHeader($key, $value);
140-
// }
141-
142-
// foreach (Arr::get($config, 'options.query_params', []) as $key => $value) {
143-
// $client->withQueryParam($key, $value);
144-
// }
145-
146-
// if ($baseUri = Arr::get($config, 'options.base_uri')) {
147-
// $client->withBaseUri($baseUri);
148-
// }
138+
$cacheEnabled = $this->config->get('cortex.llm.cache.enabled', false);
139+
$cacheStore = $cacheEnabled
140+
? $this->container->make('cache')->store($this->config->get('cortex.llm.cache.store'))
141+
: null;
149142

150143
$client = new Anthropic(
151-
Arr::get($config, 'options.api_key') ?? '',
152-
Arr::get($config, 'options.base_uri'),
153-
Arr::get($config, 'options.version'),
144+
apiKey: Arr::get($config, 'options.api_key') ?? '',
145+
baseUri: Arr::get($config, 'options.base_uri'),
146+
version: Arr::get($config, 'options.version'),
147+
cacheStore: $cacheStore,
148+
cacheExpiryInSeconds: $this->config->get('cortex.llm.cache.ttl'),
154149
);
155150

156151
if (! isset($config['default_model'])) {
@@ -169,6 +164,8 @@ public function createAnthropicDriver(array $config, string $name): AnthropicCha
169164
);
170165
$driver->setEventDispatcher(new IlluminateEventDispatcherBridge($this->container->make('events')));
171166

167+
$driver->withCaching($cacheEnabled);
168+
172169
return $driver;
173170
}
174171

‎src/SDK/Anthropic/Anthropic.php‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
use Saloon\Http\Connector;
99
use Saloon\Http\Faking\MockClient;
1010
use Saloon\Http\Auth\HeaderAuthenticator;
11+
use Illuminate\Contracts\Cache\Repository;
1112
use Cortex\SDK\Anthropic\Resources\Messages;
1213

1314
class Anthropic extends Connector
@@ -17,11 +18,17 @@ public function __construct(
1718
protected string $apiKey,
1819
protected ?string $baseUri = null,
1920
protected ?string $version = null,
21+
protected ?Repository $cacheStore = null,
22+
protected ?int $cacheExpiryInSeconds = null,
2023
) {}
2124

2225
public function messages(): Messages
2326
{
24-
return new Messages($this);
27+
return new Messages(
28+
$this,
29+
$this->cacheStore,
30+
$this->cacheExpiryInSeconds,
31+
);
2532
}
2633

2734
public function resolveBaseUrl(): string

0 commit comments

Comments
 (0)