Skip to content

Commit b891cef

Browse files
llupallupa
committed
Add ElasticSearch Store support
1 parent b9a7258 commit b891cef

File tree

16 files changed

+612
-0
lines changed

16 files changed

+612
-0
lines changed

‎examples/.env‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,5 +187,8 @@ REDIS_HOST=localhost
187187
# Manticore Search (store)
188188
MANTICORESEARCH_HOST=http://127.0.0.1:9308
189189

190+
# Elasticsearch (store)
191+
ELASTICSEARCH_ENDPOINT=http://127.0.0.1:9201
192+
190193
# OpenSearch (store)
191194
OPENSEARCH_ENDPOINT=http://127.0.0.1:9200

‎examples/commands/stores.php‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
use MongoDB\Client as MongoDbClient;
1717
use Symfony\AI\Store\Bridge\Cache\Store as CacheStore;
1818
use Symfony\AI\Store\Bridge\ClickHouse\Store as ClickHouseStore;
19+
use Symfony\AI\Store\Bridge\Elasticsearch\Store as ElasticsearchStore;
1920
use Symfony\AI\Store\Bridge\ManticoreSearch\Store as ManticoreSearchStore;
2021
use Symfony\AI\Store\Bridge\MariaDb\Store as MariaDbStore;
2122
use Symfony\AI\Store\Bridge\Meilisearch\Store as MeilisearchStore;
@@ -47,6 +48,11 @@
4748
env('CLICKHOUSE_DATABASE'),
4849
env('CLICKHOUSE_TABLE'),
4950
),
51+
'elasticsearch' => static fn (): ElasticsearchStore => new ElasticsearchStore(
52+
http_client(),
53+
env('ELASTICSEARCH_ENDPOINT'),
54+
'symfony',
55+
),
5056
'manticoresearch' => static fn (): ManticoreSearchStore => new ManticoreSearchStore(
5157
http_client(),
5258
env('MANTICORESEARCH_HOST'),

‎examples/compose.yaml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,28 @@ services:
124124
- '7474:7474'
125125
- '7687:7687'
126126

127+
elasticsearch:
128+
image: docker.elastic.co/elasticsearch/elasticsearch:9.2.2
129+
environment:
130+
discovery.type: 'single-node'
131+
xpack.security.enabled: false
132+
ES_JAVA_OPTS: '-Xms512m -Xmx512m'
133+
ulimits:
134+
memlock:
135+
soft: -1
136+
hard: -1
137+
nofile:
138+
soft: 65536
139+
hard: 65536
140+
healthcheck:
141+
test: [ 'CMD', 'curl', '-f', 'http://127.0.0.1:9201/_cluster/health' ]
142+
interval: 30s
143+
start_period: 120s
144+
timeout: 20s
145+
retries: 3
146+
ports:
147+
- '9201:9200'
148+
127149
opensearch:
128150
image: opensearchproject/opensearch
129151
environment:

‎examples/rag/elasticsearch.php‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <[email protected]>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
use Symfony\AI\Agent\Agent;
13+
use Symfony\AI\Agent\Bridge\SimilaritySearch\SimilaritySearch;
14+
use Symfony\AI\Agent\Toolbox\AgentProcessor;
15+
use Symfony\AI\Agent\Toolbox\Toolbox;
16+
use Symfony\AI\Fixtures\Movies;
17+
use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory;
18+
use Symfony\AI\Platform\Message\Message;
19+
use Symfony\AI\Platform\Message\MessageBag;
20+
use Symfony\AI\Store\Bridge\Elasticsearch\Store;
21+
use Symfony\AI\Store\Document\Loader\InMemoryLoader;
22+
use Symfony\AI\Store\Document\Metadata;
23+
use Symfony\AI\Store\Document\TextDocument;
24+
use Symfony\AI\Store\Document\Vectorizer;
25+
use Symfony\AI\Store\Indexer;
26+
use Symfony\Component\Uid\Uuid;
27+
28+
require_once dirname(__DIR__).'/bootstrap.php';
29+
30+
// initialize the store
31+
$store = new Store(
32+
httpClient: http_client(),
33+
endpoint: env('ELASTICSEARCH_ENDPOINT'),
34+
indexName: 'movies',
35+
);
36+
37+
// create embeddings and documents
38+
$documents = [];
39+
foreach (Movies::all() as $i => $movie) {
40+
$documents[] = new TextDocument(
41+
id: Uuid::v4(),
42+
content: 'Title: '.$movie['title'].\PHP_EOL.'Director: '.$movie['director'].\PHP_EOL.'Description: '.$movie['description'],
43+
metadata: new Metadata($movie),
44+
);
45+
}
46+
47+
// initialize the index
48+
$store->setup();
49+
50+
// create embeddings for documents
51+
$platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
52+
$vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
53+
$indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
54+
$indexer->index($documents);
55+
56+
$similaritySearch = new SimilaritySearch($vectorizer, $store);
57+
$toolbox = new Toolbox([$similaritySearch], logger: logger());
58+
$processor = new AgentProcessor($toolbox);
59+
$agent = new Agent($platform, 'gpt-4o-mini', [$processor], [$processor]);
60+
61+
$messages = new MessageBag(
62+
Message::forSystem('Please answer all user questions only using SimilaritySearch function.'),
63+
Message::ofUser('Which movie fits the theme of technology?')
64+
);
65+
$result = $agent->call($messages);
66+
67+
echo $result->getContent().\PHP_EOL;

‎splitsh.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
"ai-click-house-store": "src/store/src/Bridge/ClickHouse",
2828
"ai-cloudflare-store": "src/store/src/Bridge/Cloudflare",
2929
"ai-chroma-db-store": "src/store/src/Bridge/ChromaDb",
30+
"ai-elasticsearch-store": "src/store/src/Bridge/Elasticsearch",
3031
"ai-manticore-search-store": "src/store/src/Bridge/ManticoreSearch",
3132
"ai-maria-db-store": "src/store/src/Bridge/MariaDb",
3233
"ai-meilisearch-store": "src/store/src/Bridge/Meilisearch",

‎src/ai-bundle/config/options.php‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -755,6 +755,27 @@
755755
->end()
756756
->end()
757757
->end()
758+
->arrayNode('elasticsearch')
759+
->useAttributeAsKey('name')
760+
->arrayPrototype()
761+
->children()
762+
->stringNode('endpoint')->cannotBeEmpty()->end()
763+
->stringNode('index_name')->end()
764+
->stringNode('vectors_field')
765+
->defaultValue('_vectors')
766+
->end()
767+
->integerNode('dimensions')
768+
->defaultValue(1536)
769+
->end()
770+
->stringNode('similarity')
771+
->defaultValue('cosine')
772+
->end()
773+
->stringNode('http_client')
774+
->defaultValue('http_client')
775+
->end()
776+
->end()
777+
->end()
778+
->end()
758779
->arrayNode('opensearch')
759780
->useAttributeAsKey('name')
760781
->arrayPrototype()

‎src/ai-bundle/src/AiBundle.php‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@
8787
use Symfony\AI\Store\Bridge\ChromaDb\Store as ChromaDbStore;
8888
use Symfony\AI\Store\Bridge\ClickHouse\Store as ClickHouseStore;
8989
use Symfony\AI\Store\Bridge\Cloudflare\Store as CloudflareStore;
90+
use Symfony\AI\Store\Bridge\Elasticsearch\Store as ElasticsearchStore;
9091
use Symfony\AI\Store\Bridge\ManticoreSearch\Store as ManticoreSearchStore;
9192
use Symfony\AI\Store\Bridge\MariaDb\Store as MariaDbStore;
9293
use Symfony\AI\Store\Bridge\Meilisearch\Store as MeilisearchStore;
@@ -1396,6 +1397,33 @@ private function processStoreConfig(string $type, array $stores, ContainerBuilde
13961397
}
13971398
}
13981399

1400+
if ('elasticsearch' === $type) {
1401+
if (!ContainerBuilder::willBeAvailable('symfony/ai-elasticsearch-store', ElasticsearchStore::class, ['symfony/ai-bundle'])) {
1402+
throw new RuntimeException('Elasticsearch store configuration requires "symfony/ai-elasticsearch-store" package. Try running "composer require symfony/ai-elasticsearch-store".');
1403+
}
1404+
1405+
foreach ($stores as $name => $store) {
1406+
$definition = new Definition(ElasticsearchStore::class);
1407+
$definition
1408+
->setLazy(true)
1409+
->setArguments([
1410+
new Reference($store['http_client']),
1411+
$store['endpoint'],
1412+
$store['index_name'] ?? $name,
1413+
$store['vectors_field'],
1414+
$store['dimensions'],
1415+
$store['similarity'],
1416+
])
1417+
->addTag('proxy', ['interface' => StoreInterface::class])
1418+
->addTag('proxy', ['interface' => ManagedStoreInterface::class])
1419+
->addTag('ai.store');
1420+
1421+
$container->setDefinition('ai.store.'.$type.'.'.$name, $definition);
1422+
$container->registerAliasForArgument('ai.store.'.$type.'.'.$name, StoreInterface::class, $name);
1423+
$container->registerAliasForArgument('ai.store.'.$type.'.'.$name, StoreInterface::class, $type.'_'.$name);
1424+
}
1425+
}
1426+
13991427
if ('opensearch' === $type) {
14001428
if (!ContainerBuilder::willBeAvailable('symfony/ai-open-search-store', OpenSearchStore::class, ['symfony/ai-bundle'])) {
14011429
throw new RuntimeException('OpenSearch store configuration requires "symfony/ai-open-search-store" package. Try running "composer require symfony/ai-open-search-store".');

‎src/store/CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ CHANGELOG
3838
- ChromaDB
3939
- ClickHouse
4040
- Cloudflare
41+
- Elasticsearch
4142
- Manticore Search
4243
- MariaDB
4344
- Meilisearch

‎src/store/composer.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
"chromadb",
1010
"clickhouse",
1111
"cloudflare",
12+
"elasticsearch",
1213
"mariadb",
1314
"meilisearch",
1415
"milvus",

‎src/store/src/Bridge/Elasticsearch/.gitattributes‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
/Tests export-ignore
2+
/phpunit.xml.dist export-ignore
3+
/.git* export-ignore

0 commit comments

Comments
 (0)