Skip to content

Commit 9c0c1ee

Browse files
chr-hertelchr-hertel
committed
feature #105 [Store] Add Meilisearch (Guikingone)
This PR was squashed before being merged into the main branch. Discussion ---------- [Store] Add Meilisearch | Q | A | ------------- | --- | Bug fix? | no | New feature? | yes <!-- please update src/**/CHANGELOG.md files --> | Docs? | yes | Issues | Fix #93 | License | MIT Hi 👋🏻 As explained in #93, this PR aim to add `meilisearch` as a vector store, the full integration is not finished (tests to be validated) but the main implementation is ready. Commits ------- 394fd42 [Store] Add Meilisearch
2 parents acc9010 + 394fd42 commit 9c0c1ee

File tree

6 files changed

+442
-0
lines changed

6 files changed

+442
-0
lines changed

‎examples/.env‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,3 +70,7 @@ GEMINI_API_KEY=
7070

7171
# For MariaDB store. Server defined in compose.yaml
7272
MARIADB_URI=pdo-mysql://[email protected]:3309/my_database
73+
74+
# Meilisearch
75+
MEILISEARCH_HOST=http://127.0.0.1:7700
76+
MEILISEARCH_API_KEY=changeMe

‎examples/compose.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,10 @@ services:
1515
POSTGRES_PASSWORD: postgres
1616
ports:
1717
- "5432:5432"
18+
19+
meilisearch:
20+
image: getmeili/meilisearch:v1.15
21+
environment:
22+
MEILI_MASTER_KEY: "${MEILISEARCH_MASTER_KEY:-changeMe}"
23+
ports:
24+
- "7700:7700"

‎examples/store/meilisearch-similarity-search.php‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <[email protected]>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
use Symfony\AI\Agent\Agent;
13+
use Symfony\AI\Agent\Toolbox\AgentProcessor;
14+
use Symfony\AI\Agent\Toolbox\Tool\SimilaritySearch;
15+
use Symfony\AI\Agent\Toolbox\Toolbox;
16+
use Symfony\AI\Platform\Bridge\OpenAI\Embeddings;
17+
use Symfony\AI\Platform\Bridge\OpenAI\GPT;
18+
use Symfony\AI\Platform\Bridge\OpenAI\PlatformFactory;
19+
use Symfony\AI\Platform\Message\Message;
20+
use Symfony\AI\Platform\Message\MessageBag;
21+
use Symfony\AI\Store\Bridge\Meilisearch\Store;
22+
use Symfony\AI\Store\Document\Metadata;
23+
use Symfony\AI\Store\Document\TextDocument;
24+
use Symfony\AI\Store\Document\Vectorizer;
25+
use Symfony\AI\Store\Indexer;
26+
use Symfony\Component\Dotenv\Dotenv;
27+
use Symfony\Component\HttpClient\HttpClient;
28+
use Symfony\Component\Uid\Uuid;
29+
30+
require_once dirname(__DIR__).'/vendor/autoload.php';
31+
(new Dotenv())->loadEnv(dirname(__DIR__).'/.env');
32+
33+
if (!isset($_SERVER['OPENAI_API_KEY'], $_SERVER['MEILISEARCH_HOST'], $_SERVER['MEILISEARCH_API_KEY'])) {
34+
echo 'Please set OPENAI_API_KEY, MEILISEARCH_API_KEY and MEILISEARCH_HOST environment variables.'.\PHP_EOL;
35+
exit(1);
36+
}
37+
38+
// initialize the store
39+
$store = new Store(
40+
httpClient: HttpClient::create(),
41+
endpointUrl: $_SERVER['MEILISEARCH_HOST'],
42+
apiKey: $_SERVER['MEILISEARCH_API_KEY'],
43+
indexName: 'movies',
44+
);
45+
46+
// our data
47+
$movies = [
48+
['title' => 'Inception', 'description' => 'A skilled thief is given a chance at redemption if he can successfully perform inception, the act of planting an idea in someone\'s subconscious.', 'director' => 'Christopher Nolan'],
49+
['title' => 'The Matrix', 'description' => 'A hacker discovers the world he lives in is a simulated reality and joins a rebellion to overthrow its controllers.', 'director' => 'The Wachowskis'],
50+
['title' => 'The Godfather', 'description' => 'The aging patriarch of an organized crime dynasty transfers control of his empire to his reluctant son.', 'director' => 'Francis Ford Coppola'],
51+
];
52+
53+
// create embeddings and documents
54+
$documents = [];
55+
foreach ($movies as $i => $movie) {
56+
$documents[] = new TextDocument(
57+
id: Uuid::v4(),
58+
content: 'Title: '.$movie['title'].\PHP_EOL.'Director: '.$movie['director'].\PHP_EOL.'Description: '.$movie['description'],
59+
metadata: new Metadata($movie),
60+
);
61+
}
62+
63+
// initialize the index
64+
$store->initialize();
65+
66+
// create embeddings for documents
67+
$platform = PlatformFactory::create($_SERVER['OPENAI_API_KEY']);
68+
$vectorizer = new Vectorizer($platform, $embeddings = new Embeddings());
69+
$indexer = new Indexer($vectorizer, $store);
70+
$indexer->index($documents);
71+
72+
$model = new GPT(GPT::GPT_4O_MINI);
73+
74+
$similaritySearch = new SimilaritySearch($platform, $embeddings, $store);
75+
$toolbox = Toolbox::create($similaritySearch);
76+
$processor = new AgentProcessor($toolbox);
77+
$agent = new Agent($platform, $model, [$processor], [$processor]);
78+
79+
$messages = new MessageBag(
80+
Message::forSystem('Please answer all user questions only using SimilaritySearch function.'),
81+
Message::ofUser('Which movie fits the theme of technology?')
82+
);
83+
$response = $agent->call($messages);
84+
85+
echo $response->getContent().\PHP_EOL;

‎src/store/doc/index.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ You can find more advanced usage in combination with an Agent using the store fo
4040
* `Similarity Search with MariaDB (RAG)`_
4141
* `Similarity Search with MongoDB (RAG)`_
4242
* `Similarity Search with Pinecone (RAG)`_
43+
* `Similarity Search with Meilisearch (RAG)`_
4344

4445
Supported Stores
4546
----------------
@@ -50,6 +51,7 @@ Supported Stores
5051
* `MongoDB Atlas`_ (requires `mongodb/mongodb` as additional dependency)
5152
* `Pinecone`_ (requires `probots-io/pinecone-php` as additional dependency)
5253
* `Postgres`_ (requires `ext-pdo`)
54+
* `Meilisearch`_
5355

5456
.. note::
5557

@@ -86,10 +88,12 @@ This leads to a store implementing two methods::
8688
.. _`Similarity Search with MariaDB (RAG)`: https://github.com/symfony/ai/blob/main/examples/store/mariadb-similarity-search.php
8789
.. _`Similarity Search with MongoDB (RAG)`: https://github.com/symfony/ai/blob/main/examples/store/mongodb-similarity-search.php
8890
.. _`Similarity Search with Pinecone (RAG)`: https://github.com/symfony/ai/blob/main/examples/store/pinecone-similarity-search.php
91+
.. _`Similarity Search with Meilisearch (RAG)`: https://github.com/symfony/ai/blob/main/examples/store/meilisearch-similarity-search.php
8992
.. _`Azure AI Search`: https://azure.microsoft.com/products/ai-services/ai-search
9093
.. _`Chroma`: https://www.trychroma.com/
9194
.. _`MariaDB`: https://mariadb.org/projects/mariadb-vector/
9295
.. _`MongoDB Atlas`: https://www.mongodb.com/atlas
9396
.. _`Pinecone`: https://www.pinecone.io/
9497
.. _`Postgres`: https://www.postgresql.org/about/news/pgvector-070-released-2852/
98+
.. _`Meilisearch`: https://www.meilisearch.com/
9599
.. _`GitHub`: https://github.com/symfony/ai/issues/16

‎src/store/src/Bridge/Meilisearch/Store.php‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <[email protected]>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
namespace Symfony\AI\Store\Bridge\Meilisearch;
13+
14+
use Symfony\AI\Platform\Vector\NullVector;
15+
use Symfony\AI\Platform\Vector\Vector;
16+
use Symfony\AI\Store\Document\Metadata;
17+
use Symfony\AI\Store\Document\VectorDocument;
18+
use Symfony\AI\Store\Exception\InvalidArgumentException;
19+
use Symfony\AI\Store\InitializableStoreInterface;
20+
use Symfony\AI\Store\VectorStoreInterface;
21+
use Symfony\Component\Uid\Uuid;
22+
use Symfony\Contracts\HttpClient\HttpClientInterface;
23+
24+
/**
25+
* @author Guillaume Loulier <[email protected]>
26+
*/
27+
final readonly class Store implements InitializableStoreInterface, VectorStoreInterface
28+
{
29+
/**
30+
* @param string $embedder The name of the embedder where vectors are stored
31+
* @param string $vectorFieldName The name of the field int the index that contains the vector
32+
*/
33+
public function __construct(
34+
private HttpClientInterface $httpClient,
35+
private string $endpointUrl,
36+
#[\SensitiveParameter] private string $apiKey,
37+
private string $indexName,
38+
private string $embedder = 'default',
39+
private string $vectorFieldName = '_vectors',
40+
private int $embeddingsDimension = 1536,
41+
) {
42+
}
43+
44+
public function add(VectorDocument ...$documents): void
45+
{
46+
$this->request('PUT', \sprintf('indexes/%s/documents', $this->indexName), array_map(
47+
$this->convertToIndexableArray(...), $documents)
48+
);
49+
}
50+
51+
public function query(Vector $vector, array $options = [], ?float $minScore = null): array
52+
{
53+
$response = $this->request('POST', \sprintf('indexes/%s/search', $this->indexName), [
54+
'vector' => $vector->getData(),
55+
'showRankingScore' => true,
56+
'retrieveVectors' => true,
57+
'hybrid' => [
58+
'embedder' => $this->embedder,
59+
'semanticRatio' => 1.0,
60+
],
61+
]);
62+
63+
return array_map($this->convertToVectorDocument(...), $response['hits']);
64+
}
65+
66+
public function initialize(array $options = []): void
67+
{
68+
if ([] !== $options) {
69+
throw new InvalidArgumentException('No supported options');
70+
}
71+
72+
$this->request('POST', 'indexes', [
73+
'uid' => $this->indexName,
74+
'primaryKey' => 'id',
75+
]);
76+
77+
$this->request('PATCH', \sprintf('indexes/%s/settings', $this->indexName), [
78+
'embedders' => [
79+
$this->embedder => [
80+
'source' => 'userProvided',
81+
'dimensions' => $this->embeddingsDimension,
82+
],
83+
],
84+
]);
85+
}
86+
87+
/**
88+
* @param array<string, mixed> $payload
89+
*
90+
* @return array<string, mixed>
91+
*/
92+
private function request(string $method, string $endpoint, array $payload): array
93+
{
94+
$url = \sprintf('%s/%s', $this->endpointUrl, $endpoint);
95+
$response = $this->httpClient->request($method, $url, [
96+
'headers' => [
97+
'Authorization' => \sprintf('Bearer %s', $this->apiKey),
98+
],
99+
'json' => $payload,
100+
]);
101+
102+
return $response->toArray();
103+
}
104+
105+
/**
106+
* @return array<string, mixed>
107+
*/
108+
private function convertToIndexableArray(VectorDocument $document): array
109+
{
110+
return array_merge([
111+
'id' => $document->id->toRfc4122(),
112+
$this->vectorFieldName => [
113+
$this->embedder => [
114+
'embeddings' => $document->vector->getData(),
115+
'regenerate' => false,
116+
],
117+
],
118+
], $document->metadata->getArrayCopy());
119+
}
120+
121+
/**
122+
* @param array<string, mixed> $data
123+
*/
124+
private function convertToVectorDocument(array $data): VectorDocument
125+
{
126+
return new VectorDocument(
127+
id: Uuid::fromString($data['id']),
128+
vector: !\array_key_exists($this->vectorFieldName, $data) || null === $data[$this->vectorFieldName]
129+
? new NullVector()
130+
: new Vector($data[$this->vectorFieldName][$this->embedder]['embeddings']),
131+
metadata: new Metadata($data),
132+
);
133+
}
134+
}

0 commit comments

Comments
 (0)