diff --git a/docs/components/store.rst b/docs/components/store.rst index 7b518944c..e4de2f372 100644 --- a/docs/components/store.rst +++ b/docs/components/store.rst @@ -70,6 +70,7 @@ Similarity Search Examples * `Similarity Search with Milvus (RAG)`_ * `Similarity Search with MongoDB (RAG)`_ * `Similarity Search with Neo4j (RAG)`_ +* `Similarity Search with OpenSearch (RAG)`_ * `Similarity Search with Pinecone (RAG)`_ * `Similarity Search with Qdrant (RAG)`_ * `Similarity Search with SurrealDB (RAG)`_ @@ -97,6 +98,7 @@ Supported Stores * `Milvus`_ * `MongoDB Atlas`_ (requires ``mongodb/mongodb`` as additional dependency) * `Neo4j`_ +* `OpenSearch`_ * `Pinecone`_ (requires ``probots-io/pinecone-php`` as additional dependency) * `Postgres`_ (requires ``ext-pdo``) * `Qdrant`_ @@ -165,6 +167,7 @@ This leads to a store implementing two methods:: .. _`Similarity Search with Milvus (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/milvus.php .. _`Similarity Search with MongoDB (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/mongodb.php .. _`Similarity Search with Neo4j (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/neo4j.php +.. _`Similarity Search with OpenSearch (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/opensearch.php .. _`Similarity Search with Pinecone (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/pinecone.php .. _`Similarity Search with Symfony Cache (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/cache.php .. _`Similarity Search with Qdrant (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/qdrant.php @@ -186,6 +189,7 @@ This leads to a store implementing two methods:: .. _`InMemory`: https://www.php.net/manual/en/language.types.array.php .. _`Qdrant`: https://qdrant.tech/ .. _`Neo4j`: https://neo4j.com/ +.. _`OpenSearch`: https://opensearch.org/ .. _`Typesense`: https://typesense.org/ .. _`Symfony Cache`: https://symfony.com/doc/current/components/cache.html .. _`Weaviate`: https://weaviate.io/ diff --git a/examples/.env b/examples/.env index d12133301..0900bcc5a 100644 --- a/examples/.env +++ b/examples/.env @@ -182,3 +182,6 @@ REDIS_HOST=localhost # Manticore (store) MANTICORE_HOST=http://127.0.0.1:9308 + +# OpenSearch (store) +OPENSEARCH_ENDPOINT=http://127.0.0.1:9200 diff --git a/examples/commands/message-stores.php b/examples/commands/message-stores.php index 90ef1cd21..841583e34 100644 --- a/examples/commands/message-stores.php +++ b/examples/commands/message-stores.php @@ -103,6 +103,9 @@ new DropStoreCommand(new ServiceLocator($factories)), ]); +$clock = new MonotonicClock(); +$clock->sleep(10); + foreach ($storesIds as $store) { $setupOutputCode = $application->run(new ArrayInput([ 'command' => 'ai:message-store:setup', diff --git a/examples/commands/stores.php b/examples/commands/stores.php index 29b25bfbb..237d9c2a0 100644 --- a/examples/commands/stores.php +++ b/examples/commands/stores.php @@ -23,6 +23,7 @@ use Symfony\AI\Store\Bridge\Milvus\Store as MilvusStore; use Symfony\AI\Store\Bridge\MongoDb\Store as MongoDbStore; use Symfony\AI\Store\Bridge\Neo4j\Store as Neo4jStore; +use Symfony\AI\Store\Bridge\OpenSearch\Store as OpenSearchStore; use Symfony\AI\Store\Bridge\Postgres\Store as PostgresStore; use Symfony\AI\Store\Bridge\Qdrant\Store as QdrantStore; use Symfony\AI\Store\Bridge\Redis\Store as RedisStore; @@ -32,6 +33,7 @@ use Symfony\AI\Store\Command\DropStoreCommand; use Symfony\AI\Store\Command\SetupStoreCommand; use Symfony\Component\Cache\Adapter\ArrayAdapter; +use Symfony\Component\Clock\MonotonicClock; use Symfony\Component\Console\Application; use Symfony\Component\Console\Input\ArrayInput; use Symfony\Component\Console\Output\ConsoleOutput; @@ -86,6 +88,11 @@ vectorIndexName: 'Commands', nodeName: 'symfony', ), + 'opensearch' => static fn (): OpenSearchStore => new OpenSearchStore( + http_client(), + env('OPENSEARCH_ENDPOINT'), + 'symfony', + ), 'postgres' => static fn (): PostgresStore => PostgresStore::fromDbal( DriverManager::getConnection((new DsnParser())->parse(env('POSTGRES_URI'))), 'my_table', @@ -133,6 +140,9 @@ new DropStoreCommand(new ServiceLocator($factories)), ]); +$clock = new MonotonicClock(); +$clock->sleep(10); + foreach ($storesIds as $store) { $setupOutputCode = $application->run(new ArrayInput([ 'command' => 'ai:store:setup', diff --git a/examples/compose.yaml b/examples/compose.yaml index 46902a42e..4a1a17fa6 100644 --- a/examples/compose.yaml +++ b/examples/compose.yaml @@ -124,6 +124,31 @@ services: - '7474:7474' - '7687:7687' + opensearch: + image: opensearchproject/opensearch + environment: + discovery.type: 'single-node' + bootstrap.memory_lock: true + indices.requests.cache.maximum_cacheable_size: 256 + DISABLE_SECURITY_PLUGIN: true + OPENSEARCH_JAVA_OPTS: '-Xms512m -Xmx512m' + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + healthcheck: + test: [ 'CMD', 'curl', "-f", "http://127.0.0.1:9200" ] + interval: 30s + start_period: 120s + timeout: 20s + retries: 3 + ports: + - '9200:9200' + - '9600:9600' + pogocache: image: pogocache/pogocache command: [ 'pogocache', '--auth', 'symfony' ] diff --git a/examples/rag/opensearch.php b/examples/rag/opensearch.php new file mode 100644 index 000000000..a23517fb5 --- /dev/null +++ b/examples/rag/opensearch.php @@ -0,0 +1,67 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +use Symfony\AI\Agent\Agent; +use Symfony\AI\Agent\Bridge\SimilaritySearch\SimilaritySearch; +use Symfony\AI\Agent\Toolbox\AgentProcessor; +use Symfony\AI\Agent\Toolbox\Toolbox; +use Symfony\AI\Fixtures\Movies; +use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory; +use Symfony\AI\Platform\Message\Message; +use Symfony\AI\Platform\Message\MessageBag; +use Symfony\AI\Store\Bridge\OpenSearch\Store; +use Symfony\AI\Store\Document\Loader\InMemoryLoader; +use Symfony\AI\Store\Document\Metadata; +use Symfony\AI\Store\Document\TextDocument; +use Symfony\AI\Store\Document\Vectorizer; +use Symfony\AI\Store\Indexer; +use Symfony\Component\Uid\Uuid; + +require_once dirname(__DIR__).'/bootstrap.php'; + +// initialize the store +$store = new Store( + httpClient: http_client(), + endpoint: env('OPENSEARCH_ENDPOINT'), + indexName: 'movies', +); + +// create embeddings and documents +$documents = []; +foreach (Movies::all() as $i => $movie) { + $documents[] = new TextDocument( + id: Uuid::v4(), + content: 'Title: '.$movie['title'].\PHP_EOL.'Director: '.$movie['director'].\PHP_EOL.'Description: '.$movie['description'], + metadata: new Metadata($movie), + ); +} + +// initialize the index +$store->setup(); + +// create embeddings for documents +$platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client()); +$vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger()); +$indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger()); +$indexer->index($documents); + +$similaritySearch = new SimilaritySearch($vectorizer, $store); +$toolbox = new Toolbox([$similaritySearch], logger: logger()); +$processor = new AgentProcessor($toolbox); +$agent = new Agent($platform, 'gpt-4o-mini', [$processor], [$processor]); + +$messages = new MessageBag( + Message::forSystem('Please answer all user questions only using SimilaritySearch function.'), + Message::ofUser('Which movie fits the theme of technology?') +); +$result = $agent->call($messages); + +echo $result->getContent().\PHP_EOL; diff --git a/src/ai-bundle/config/options.php b/src/ai-bundle/config/options.php index 0bef84431..f9c7bbbde 100644 --- a/src/ai-bundle/config/options.php +++ b/src/ai-bundle/config/options.php @@ -722,6 +722,27 @@ ->end() ->end() ->end() + ->arrayNode('opensearch') + ->useAttributeAsKey('name') + ->arrayPrototype() + ->children() + ->stringNode('endpoint')->cannotBeEmpty()->end() + ->stringNode('index_name')->end() + ->stringNode('vectors_field') + ->defaultValue('_vectors') + ->end() + ->integerNode('dimensions') + ->defaultValue(1536) + ->end() + ->stringNode('space_type') + ->defaultValue('l2') + ->end() + ->stringNode('http_client') + ->defaultValue('http_client') + ->end() + ->end() + ->end() + ->end() ->arrayNode('pinecone') ->useAttributeAsKey('name') ->arrayPrototype() diff --git a/src/ai-bundle/src/AiBundle.php b/src/ai-bundle/src/AiBundle.php index 7246667a2..4cd39f910 100644 --- a/src/ai-bundle/src/AiBundle.php +++ b/src/ai-bundle/src/AiBundle.php @@ -91,6 +91,7 @@ use Symfony\AI\Store\Bridge\Milvus\Store as MilvusStore; use Symfony\AI\Store\Bridge\MongoDb\Store as MongoDbStore; use Symfony\AI\Store\Bridge\Neo4j\Store as Neo4jStore; +use Symfony\AI\Store\Bridge\OpenSearch\Store as OpenSearchStore; use Symfony\AI\Store\Bridge\Pinecone\Store as PineconeStore; use Symfony\AI\Store\Bridge\Postgres\Store as PostgresStore; use Symfony\AI\Store\Bridge\Qdrant\Store as QdrantStore; @@ -1277,6 +1278,29 @@ private function processStoreConfig(string $type, array $stores, ContainerBuilde } } + if ('opensearch' === $type) { + foreach ($stores as $name => $store) { + $definition = new Definition(OpenSearchStore::class); + $definition + ->setLazy(true) + ->setArguments([ + new Reference($store['http_client']), + $store['endpoint'], + $store['index_name'] ?? $name, + $store['vectors_field'], + $store['dimensions'], + $store['space_type'], + ]) + ->addTag('proxy', ['interface' => StoreInterface::class]) + ->addTag('proxy', ['interface' => ManagedStoreInterface::class]) + ->addTag('ai.store'); + + $container->setDefinition('ai.store.'.$type.'.'.$name, $definition); + $container->registerAliasForArgument('ai.store.'.$type.'.'.$name, StoreInterface::class, $name); + $container->registerAliasForArgument('ai.store.'.$type.'.'.$name, StoreInterface::class, $type.'_'.$name); + } + } + if ('pinecone' === $type) { foreach ($stores as $name => $store) { $arguments = [ diff --git a/src/ai-bundle/tests/DependencyInjection/AiBundleTest.php b/src/ai-bundle/tests/DependencyInjection/AiBundleTest.php index af64384b8..848926dc1 100644 --- a/src/ai-bundle/tests/DependencyInjection/AiBundleTest.php +++ b/src/ai-bundle/tests/DependencyInjection/AiBundleTest.php @@ -47,6 +47,7 @@ use Symfony\AI\Store\Bridge\Milvus\Store as MilvusStore; use Symfony\AI\Store\Bridge\MongoDb\Store as MongoDbStore; use Symfony\AI\Store\Bridge\Neo4j\Store as Neo4jStore; +use Symfony\AI\Store\Bridge\OpenSearch\Store as OpenSearchStore; use Symfony\AI\Store\Bridge\Pinecone\Store as PineconeStore; use Symfony\AI\Store\Bridge\Postgres\Distance; use Symfony\AI\Store\Bridge\Postgres\Store as PostgresStore; @@ -2076,6 +2077,269 @@ public function testNeo4jStoreWithQuantizationCanBeConfigured() $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface')); } + public function testOpenSearchStoreCanBeConfigured() + { + $container = $this->buildContainer([ + 'ai' => [ + 'store' => [ + 'opensearch' => [ + 'my_opensearch_store' => [ + 'endpoint' => 'http://127.0.0.1:9200', + ], + ], + ], + ], + ]); + + $this->assertTrue($container->hasDefinition('ai.store.opensearch.my_opensearch_store')); + + $definition = $container->getDefinition('ai.store.opensearch.my_opensearch_store'); + $this->assertSame(OpenSearchStore::class, $definition->getClass()); + + $this->assertTrue($definition->isLazy()); + $this->assertCount(6, $definition->getArguments()); + $this->assertInstanceOf(Reference::class, $definition->getArgument(0)); + $this->assertSame('http_client', (string) $definition->getArgument(0)); + $this->assertSame('http://127.0.0.1:9200', $definition->getArgument(1)); + $this->assertSame('my_opensearch_store', $definition->getArgument(2)); + $this->assertSame('_vectors', $definition->getArgument(3)); + $this->assertSame(1536, $definition->getArgument(4)); + $this->assertSame('l2', $definition->getArgument(5)); + + $this->assertTrue($definition->hasTag('proxy')); + $this->assertSame([ + ['interface' => StoreInterface::class], + ['interface' => ManagedStoreInterface::class], + ], $definition->getTag('proxy')); + $this->assertTrue($definition->hasTag('ai.store')); + + $this->assertTrue($container->hasAlias('.Symfony\AI\Store\StoreInterface $my_opensearch_store')); + $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface $myOpensearchStore')); + $this->assertTrue($container->hasAlias('.Symfony\AI\Store\StoreInterface $opensearch_my_opensearch_store')); + $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface $opensearchMyOpensearchStore')); + $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface')); + } + + public function testOpenSearchStoreWithCustomIndexCanBeConfigured() + { + $container = $this->buildContainer([ + 'ai' => [ + 'store' => [ + 'opensearch' => [ + 'my_opensearch_store' => [ + 'endpoint' => 'http://127.0.0.1:9200', + 'index_name' => 'foo', + ], + ], + ], + ], + ]); + + $this->assertTrue($container->hasDefinition('ai.store.opensearch.my_opensearch_store')); + + $definition = $container->getDefinition('ai.store.opensearch.my_opensearch_store'); + $this->assertSame(OpenSearchStore::class, $definition->getClass()); + + $this->assertTrue($definition->isLazy()); + $this->assertCount(6, $definition->getArguments()); + $this->assertInstanceOf(Reference::class, $definition->getArgument(0)); + $this->assertSame('http_client', (string) $definition->getArgument(0)); + $this->assertSame('http://127.0.0.1:9200', $definition->getArgument(1)); + $this->assertSame('foo', $definition->getArgument(2)); + $this->assertSame('_vectors', $definition->getArgument(3)); + $this->assertSame(1536, $definition->getArgument(4)); + $this->assertSame('l2', $definition->getArgument(5)); + + $this->assertTrue($definition->hasTag('proxy')); + $this->assertSame([ + ['interface' => StoreInterface::class], + ['interface' => ManagedStoreInterface::class], + ], $definition->getTag('proxy')); + $this->assertTrue($definition->hasTag('ai.store')); + + $this->assertTrue($container->hasAlias('.Symfony\AI\Store\StoreInterface $my_opensearch_store')); + $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface $myOpensearchStore')); + $this->assertTrue($container->hasAlias('.Symfony\AI\Store\StoreInterface $opensearch_my_opensearch_store')); + $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface $opensearchMyOpensearchStore')); + $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface')); + } + + public function testOpenSearchStoreWithCustomFieldCanBeConfigured() + { + $container = $this->buildContainer([ + 'ai' => [ + 'store' => [ + 'opensearch' => [ + 'my_opensearch_store' => [ + 'endpoint' => 'http://127.0.0.1:9200', + 'vectors_field' => 'foo', + ], + ], + ], + ], + ]); + + $this->assertTrue($container->hasDefinition('ai.store.opensearch.my_opensearch_store')); + + $definition = $container->getDefinition('ai.store.opensearch.my_opensearch_store'); + $this->assertSame(OpenSearchStore::class, $definition->getClass()); + + $this->assertTrue($definition->isLazy()); + $this->assertCount(6, $definition->getArguments()); + $this->assertInstanceOf(Reference::class, $definition->getArgument(0)); + $this->assertSame('http_client', (string) $definition->getArgument(0)); + $this->assertSame('http://127.0.0.1:9200', $definition->getArgument(1)); + $this->assertSame('my_opensearch_store', $definition->getArgument(2)); + $this->assertSame('foo', $definition->getArgument(3)); + $this->assertSame(1536, $definition->getArgument(4)); + $this->assertSame('l2', $definition->getArgument(5)); + + $this->assertTrue($definition->hasTag('proxy')); + $this->assertSame([ + ['interface' => StoreInterface::class], + ['interface' => ManagedStoreInterface::class], + ], $definition->getTag('proxy')); + $this->assertTrue($definition->hasTag('ai.store')); + + $this->assertTrue($container->hasAlias('.Symfony\AI\Store\StoreInterface $my_opensearch_store')); + $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface $myOpensearchStore')); + $this->assertTrue($container->hasAlias('.Symfony\AI\Store\StoreInterface $opensearch_my_opensearch_store')); + $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface $opensearchMyOpensearchStore')); + $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface')); + } + + public function testOpenSearchStoreWithCustomDimensionsCanBeConfigured() + { + $container = $this->buildContainer([ + 'ai' => [ + 'store' => [ + 'opensearch' => [ + 'my_opensearch_store' => [ + 'endpoint' => 'http://127.0.0.1:9200', + 'dimensions' => 768, + ], + ], + ], + ], + ]); + + $this->assertTrue($container->hasDefinition('ai.store.opensearch.my_opensearch_store')); + + $definition = $container->getDefinition('ai.store.opensearch.my_opensearch_store'); + $this->assertSame(OpenSearchStore::class, $definition->getClass()); + + $this->assertTrue($definition->isLazy()); + $this->assertCount(6, $definition->getArguments()); + $this->assertInstanceOf(Reference::class, $definition->getArgument(0)); + $this->assertSame('http_client', (string) $definition->getArgument(0)); + $this->assertSame('http://127.0.0.1:9200', $definition->getArgument(1)); + $this->assertSame('my_opensearch_store', $definition->getArgument(2)); + $this->assertSame('_vectors', $definition->getArgument(3)); + $this->assertSame(768, $definition->getArgument(4)); + $this->assertSame('l2', $definition->getArgument(5)); + + $this->assertTrue($definition->hasTag('proxy')); + $this->assertSame([ + ['interface' => StoreInterface::class], + ['interface' => ManagedStoreInterface::class], + ], $definition->getTag('proxy')); + $this->assertTrue($definition->hasTag('ai.store')); + + $this->assertTrue($container->hasAlias('.Symfony\AI\Store\StoreInterface $my_opensearch_store')); + $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface $myOpensearchStore')); + $this->assertTrue($container->hasAlias('.Symfony\AI\Store\StoreInterface $opensearch_my_opensearch_store')); + $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface $opensearchMyOpensearchStore')); + $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface')); + } + + public function testOpenSearchStoreWithCustomSpaceTypeCanBeConfigured() + { + $container = $this->buildContainer([ + 'ai' => [ + 'store' => [ + 'opensearch' => [ + 'my_opensearch_store' => [ + 'endpoint' => 'http://127.0.0.1:9200', + 'space_type' => 'l1', + ], + ], + ], + ], + ]); + + $this->assertTrue($container->hasDefinition('ai.store.opensearch.my_opensearch_store')); + + $definition = $container->getDefinition('ai.store.opensearch.my_opensearch_store'); + $this->assertSame(OpenSearchStore::class, $definition->getClass()); + + $this->assertTrue($definition->isLazy()); + $this->assertCount(6, $definition->getArguments()); + $this->assertInstanceOf(Reference::class, $definition->getArgument(0)); + $this->assertSame('http_client', (string) $definition->getArgument(0)); + $this->assertSame('http://127.0.0.1:9200', $definition->getArgument(1)); + $this->assertSame('my_opensearch_store', $definition->getArgument(2)); + $this->assertSame('_vectors', $definition->getArgument(3)); + $this->assertSame(1536, $definition->getArgument(4)); + $this->assertSame('l1', $definition->getArgument(5)); + + $this->assertTrue($definition->hasTag('proxy')); + $this->assertSame([ + ['interface' => StoreInterface::class], + ['interface' => ManagedStoreInterface::class], + ], $definition->getTag('proxy')); + $this->assertTrue($definition->hasTag('ai.store')); + + $this->assertTrue($container->hasAlias('.Symfony\AI\Store\StoreInterface $my_opensearch_store')); + $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface $myOpensearchStore')); + $this->assertTrue($container->hasAlias('.Symfony\AI\Store\StoreInterface $opensearch_my_opensearch_store')); + $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface $opensearchMyOpensearchStore')); + $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface')); + } + + public function testOpenSearchStoreWithCustomHttpClientCanBeConfigured() + { + $container = $this->buildContainer([ + 'ai' => [ + 'store' => [ + 'opensearch' => [ + 'my_opensearch_store' => [ + 'endpoint' => 'http://127.0.0.1:9200', + 'http_client' => 'foo', + ], + ], + ], + ], + ]); + + $this->assertTrue($container->hasDefinition('ai.store.opensearch.my_opensearch_store')); + + $definition = $container->getDefinition('ai.store.opensearch.my_opensearch_store'); + $this->assertSame(OpenSearchStore::class, $definition->getClass()); + + $this->assertTrue($definition->isLazy()); + $this->assertCount(6, $definition->getArguments()); + $this->assertInstanceOf(Reference::class, $definition->getArgument(0)); + $this->assertSame('foo', (string) $definition->getArgument(0)); + $this->assertSame('http://127.0.0.1:9200', $definition->getArgument(1)); + $this->assertSame('my_opensearch_store', $definition->getArgument(2)); + $this->assertSame('_vectors', $definition->getArgument(3)); + $this->assertSame(1536, $definition->getArgument(4)); + $this->assertSame('l2', $definition->getArgument(5)); + + $this->assertTrue($definition->hasTag('proxy')); + $this->assertSame([ + ['interface' => StoreInterface::class], + ['interface' => ManagedStoreInterface::class], + ], $definition->getTag('proxy')); + $this->assertTrue($definition->hasTag('ai.store')); + + $this->assertTrue($container->hasAlias('.Symfony\AI\Store\StoreInterface $my_opensearch_store')); + $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface $myOpensearchStore')); + $this->assertTrue($container->hasAlias('.Symfony\AI\Store\StoreInterface $opensearch_my_opensearch_store')); + $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface $opensearchMyOpensearchStore')); + $this->assertTrue($container->hasAlias('Symfony\AI\Store\StoreInterface')); + } + public function testPineconeStoreCanBeConfigured() { $container = $this->buildContainer([ @@ -6856,6 +7120,27 @@ private function getFullConfig(): array 'quantization' => true, ], ], + 'opensearch' => [ + 'my_opensearch_store' => [ + 'endpoint' => 'http://127.0.0.1:9200', + ], + 'my_opensearch_store_with_custom_index' => [ + 'endpoint' => 'http://127.0.0.1:9200', + 'index_name' => 'foo', + ], + 'my_opensearch_store_with_custom_field' => [ + 'endpoint' => 'http://127.0.0.1:9200', + 'vectors_field' => 'foo', + ], + 'my_opensearch_store_with_custom_space_type' => [ + 'endpoint' => 'http://127.0.0.1:9200', + 'space_type' => 'l1', + ], + 'my_opensearch_store_with_custom_http_client' => [ + 'endpoint' => 'http://127.0.0.1:9200', + 'http_client' => 'foo', + ], + ], 'pinecone' => [ 'my_pinecone_store' => [ 'namespace' => 'my_namespace', diff --git a/src/store/CHANGELOG.md b/src/store/CHANGELOG.md index 1b2c0bff0..a5a08772c 100644 --- a/src/store/CHANGELOG.md +++ b/src/store/CHANGELOG.md @@ -43,6 +43,7 @@ CHANGELOG - Meilisearch - MongoDB - Neo4j + - OpenSearch - Pinecone - PostgreSQL with pgvector extension - Qdrant diff --git a/src/store/composer.json b/src/store/composer.json index 1eb1cdac1..d43d63e43 100644 --- a/src/store/composer.json +++ b/src/store/composer.json @@ -14,6 +14,7 @@ "milvus", "mongodb", "neo4j", + "opensearch", "pinecone", "postgres", "qdrant", diff --git a/src/store/src/Bridge/OpenSearch/Store.php b/src/store/src/Bridge/OpenSearch/Store.php new file mode 100644 index 000000000..4fc658e8b --- /dev/null +++ b/src/store/src/Bridge/OpenSearch/Store.php @@ -0,0 +1,158 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\AI\Store\Bridge\OpenSearch; + +use Symfony\AI\Platform\Vector\NullVector; +use Symfony\AI\Platform\Vector\Vector; +use Symfony\AI\Store\Document\Metadata; +use Symfony\AI\Store\Document\VectorDocument; +use Symfony\AI\Store\Exception\InvalidArgumentException; +use Symfony\AI\Store\ManagedStoreInterface; +use Symfony\AI\Store\StoreInterface; +use Symfony\Component\Uid\Uuid; +use Symfony\Contracts\HttpClient\HttpClientInterface; + +/** + * @author Guillaume Loulier + */ +final class Store implements ManagedStoreInterface, StoreInterface +{ + public function __construct( + private readonly HttpClientInterface $httpClient, + private readonly string $endpoint, + private readonly string $indexName, + private readonly string $vectorsField = '_vectors', + private readonly int $dimensions = 1536, + private readonly string $spaceType = 'l2', + ) { + } + + public function setup(array $options = []): void + { + $indexExistResponse = $this->httpClient->request('HEAD', \sprintf('%s/%s', $this->endpoint, $this->indexName)); + + if (200 === $indexExistResponse->getStatusCode()) { + return; + } + + $this->request('PUT', $this->indexName, [ + 'settings' => [ + 'index.knn' => true, + ], + 'mappings' => [ + 'properties' => [ + $this->vectorsField => [ + 'type' => 'knn_vector', + 'dimension' => $options['dimensions'] ?? $this->dimensions, + 'space_type' => $options['space_type'] ?? $this->spaceType, + ], + ], + ], + ]); + } + + public function drop(): void + { + $indexExistResponse = $this->httpClient->request('HEAD', \sprintf('%s/%s', $this->endpoint, $this->indexName)); + + if (404 === $indexExistResponse->getStatusCode()) { + throw new InvalidArgumentException(\sprintf('The index "%s" does not exist.', $this->indexName)); + } + + $this->request('DELETE', $this->indexName); + } + + public function add(VectorDocument ...$documents): void + { + $documentToIndex = fn (VectorDocument $document): array => [ + 'index' => [ + '_index' => $this->indexName, + '_id' => $document->id->toRfc4122(), + ], + ]; + + $documentToPayload = fn (VectorDocument $document): array => [ + $this->vectorsField => $document->vector->getData(), + 'metadata' => json_encode($document->metadata->getArrayCopy()), + ]; + + $this->request('POST', '_bulk', function () use ($documents, $documentToIndex, $documentToPayload) { + foreach ($documents as $document) { + yield json_encode($documentToIndex($document)).\PHP_EOL.json_encode($documentToPayload($document)).\PHP_EOL; + } + }); + } + + public function query(Vector $vector, array $options = []): iterable + { + $documents = $this->request('POST', \sprintf('%s/_search', $this->indexName), [ + 'size' => $options['size'] ?? 100, + 'query' => [ + 'knn' => [ + $this->vectorsField => [ + 'vector' => $vector->getData(), + 'k' => $options['k'] ?? 100, + ], + ], + ], + ]); + + foreach ($documents['hits']['hits'] as $document) { + yield $this->convertToVectorDocument($document); + } + } + + /** + * @param \Closure|array $payload + * + * @return array + */ + private function request(string $method, string $path, \Closure|array $payload = []): array + { + $finalOptions = []; + + if (\is_array($payload) && [] !== $payload) { + $finalOptions['json'] = $payload; + } + + if ($payload instanceof \Closure) { + $finalOptions = [ + 'headers' => [ + 'Content-Type' => 'application/x-ndjson', + ], + 'body' => $payload(), + ]; + } + + $response = $this->httpClient->request($method, \sprintf('%s/%s', $this->endpoint, $path), $finalOptions); + + return $response->toArray(); + } + + /** + * @param array{ + * '_id'?: string, + * '_source': array, + * '_score': float, + * } $document + */ + private function convertToVectorDocument(array $document): VectorDocument + { + $id = $document['_id'] ?? throw new InvalidArgumentException('Missing "_id" field in the document data.'); + + $vector = !\array_key_exists($this->vectorsField, $document['_source']) || null === $document['_source'][$this->vectorsField] + ? new NullVector() + : new Vector($document['_source'][$this->vectorsField]); + + return new VectorDocument(Uuid::fromString($id), $vector, new Metadata(json_decode($document['_source']['metadata'], true)), $document['_score'] ?? null); + } +} diff --git a/src/store/tests/Bridge/OpenSearch/StoreTest.php b/src/store/tests/Bridge/OpenSearch/StoreTest.php new file mode 100644 index 000000000..774a47f8b --- /dev/null +++ b/src/store/tests/Bridge/OpenSearch/StoreTest.php @@ -0,0 +1,198 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\AI\Store\Tests\Bridge\OpenSearch; + +use PHPUnit\Framework\TestCase; +use Symfony\AI\Platform\Vector\Vector; +use Symfony\AI\Store\Bridge\OpenSearch\Store; +use Symfony\AI\Store\Document\VectorDocument; +use Symfony\AI\Store\Exception\InvalidArgumentException; +use Symfony\Component\HttpClient\MockHttpClient; +use Symfony\Component\HttpClient\Response\JsonMockResponse; +use Symfony\Component\Uid\Uuid; + +final class StoreTest extends TestCase +{ + public function testStoreCannotSetupOnExistingIndex() + { + $httpClient = new MockHttpClient([ + new JsonMockResponse('', [ + 'http_code' => 200, + ]), + ]); + + $store = new Store($httpClient, 'http://127.0.0.1:9200', 'foo'); + $store->setup(); + + $this->assertSame(1, $httpClient->getRequestsCount()); + } + + public function testStoreCanSetup() + { + $httpClient = new MockHttpClient([ + new JsonMockResponse('', [ + 'http_code' => 404, + ]), + new JsonMockResponse([ + 'settings' => [], + 'mappings' => [], + 'aliases' => [], + ], [ + 'http_code' => 200, + ]), + ]); + + $store = new Store($httpClient, 'http://127.0.0.1:9200', 'foo'); + $store->setup(); + + $this->assertSame(2, $httpClient->getRequestsCount()); + } + + public function testStoreCanSetupWithExtraOptions() + { + $httpClient = new MockHttpClient([ + new JsonMockResponse('', [ + 'http_code' => 400, + ]), + new JsonMockResponse([ + 'settings' => [], + 'mappings' => [], + 'aliases' => [], + ], [ + 'http_code' => 200, + ]), + ]); + + $store = new Store($httpClient, 'http://127.0.0.1:9200', 'foo'); + + $store->setup([ + 'dimensions' => 768, + 'space_type' => 'l1', + ]); + + $this->assertSame(2, $httpClient->getRequestsCount()); + } + + public function testStoreCannotDropOnUndefinedIndex() + { + $httpClient = new MockHttpClient([ + new JsonMockResponse('', [ + 'http_code' => 404, + ]), + ]); + + $store = new Store($httpClient, 'http://127.0.0.1:9200', 'foo'); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('The index "foo" does not exist.'); + $this->expectExceptionCode(0); + $store->drop(); + } + + public function testStoreCanDrop() + { + $httpClient = new MockHttpClient([ + new JsonMockResponse('', [ + 'http_code' => 200, + ]), + new JsonMockResponse([ + 'acknowledged' => true, + 'shards_acknowledged' => true, + 'index' => 'foo', + ], [ + 'http_code' => 200, + ]), + ]); + + $store = new Store($httpClient, 'http://127.0.0.1:9200', 'foo'); + $store->drop(); + + $this->assertSame(2, $httpClient->getRequestsCount()); + } + + public function testStoreCanSave() + { + $httpClient = new MockHttpClient([ + new JsonMockResponse([ + 'took' => 100, + 'errors' => false, + 'items' => [ + [ + 'index' => [ + '_index' => 'foo', + '_id' => Uuid::v7()->toRfc4122(), + '_version' => 1, + 'result' => 'created', + '_shards' => [], + 'status' => 201, + ], + ], + ], + ], [ + 'http_code' => 200, + ]), + ]); + + $store = new Store($httpClient, 'http://127.0.0.1:9200', 'foo'); + $store->add(new VectorDocument(Uuid::v7(), new Vector([0.1, 0.2, 0.3]))); + + $this->assertSame(1, $httpClient->getRequestsCount()); + } + + public function testStoreCanQuery() + { + $httpClient = new MockHttpClient([ + new JsonMockResponse([ + 'took' => 100, + 'errors' => false, + 'hits' => [ + 'total' => [ + 'value' => 1, + 'relation' => 'eq', + ], + 'hits' => [ + [ + '_index' => 'foo', + '_id' => Uuid::v7()->toRfc4122(), + '_score' => 0.4363918, + '_source' => [ + '_vectors' => [0.1, 0.2, 0.3], + 'metadata' => json_encode([ + 'foo' => 'bar', + ]), + ], + ], + [ + '_index' => 'foo', + '_id' => Uuid::v7()->toRfc4122(), + '_score' => 0.4363918, + '_source' => [ + '_vectors' => [0.1, 0.4, 0.3], + 'metadata' => json_encode([ + 'foo' => 'bar', + ]), + ], + ], + ], + ], + ], [ + 'http_code' => 200, + ]), + ]); + + $store = new Store($httpClient, 'http://127.0.0.1:9200', 'foo'); + $results = $store->query(new Vector([0.1, 0.2, 0.3])); + + $this->assertCount(2, iterator_to_array($results)); + $this->assertSame(1, $httpClient->getRequestsCount()); + } +}