Skip to content

Commit f22d8ec

Browse files
GromNaNGromNaN
authored
Add SchemaManager::waitForSearchIndex() (#2830)
1 parent 18a3c34 commit f22d8ec

File tree

8 files changed

+212
-45
lines changed

8 files changed

+212
-45
lines changed

‎lib/Doctrine/ODM/MongoDB/SchemaException.php‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace Doctrine\ODM\MongoDB;
6+
7+
use RuntimeException;
8+
9+
use function implode;
10+
use function sprintf;
11+
12+
/**
13+
* Exception for schema-related issues.
14+
*/
15+
final class SchemaException extends RuntimeException
16+
{
17+
/**
18+
* @internal
19+
*
20+
* @param string[] $missingIndexes
21+
*/
22+
public static function missingSearchIndex(string $documentClass, array $missingIndexes): self
23+
{
24+
return new self(sprintf('The document class "%s" is missing the following search index(es): "%s"', $documentClass, implode('", "', $missingIndexes)));
25+
}
26+
}

‎lib/Doctrine/ODM/MongoDB/SchemaManager.php‎

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,14 @@
2020
use function array_diff_key;
2121
use function array_filter;
2222
use function array_keys;
23+
use function array_map;
2324
use function array_merge;
2425
use function array_search;
2526
use function array_unique;
2627
use function array_values;
2728
use function assert;
2829
use function count;
30+
use function hrtime;
2931
use function implode;
3032
use function in_array;
3133
use function is_array;
@@ -35,6 +37,7 @@
3537
use function ksort;
3638
use function sprintf;
3739
use function str_contains;
40+
use function usleep;
3841

3942
/**
4043
* @phpstan-import-type IndexMapping from ClassMetadata
@@ -337,6 +340,65 @@ public function createSearchIndexes(): void
337340
}
338341
}
339342

343+
/**
344+
* Wait until all search indexes are queryable for the given document classes.
345+
*
346+
* @param list<class-string>|null $classNames List of class names to check, or null to check all mapped classes
347+
* @param positive-int $maxTimeMs Maximum time to wait in milliseconds (default: 10,000 ms)
348+
* @param positive-int $waitTimeMs Time to wait between checks in milliseconds (default: 100 ms)
349+
*/
350+
public function waitForSearchIndexes(?array $classNames = null, int $maxTimeMs = 10_000, int $waitTimeMs = 100): void
351+
{
352+
if ($maxTimeMs < 1) {
353+
throw new InvalidArgumentException('$maxTimeMs must be a positive number of milliseconds.');
354+
}
355+
356+
if ($waitTimeMs < 1) {
357+
throw new InvalidArgumentException('$waitTimeMs must be a positive number of milliseconds.');
358+
}
359+
360+
$classes = $classNames === null ? $this->metadataFactory->getAllMetadata() : array_map($this->metadataFactory->getMetadataFor(...), $classNames);
361+
362+
/** @var array<class-string, string[]> $indexesToCheck Search indexes for each class */
363+
$indexesToCheck = [];
364+
foreach ($classes as $class) {
365+
if (! $class->hasSearchIndexes()) {
366+
continue;
367+
}
368+
369+
$indexesToCheck[$class->getName()] = array_column($class->getSearchIndexes(), 'name');
370+
}
371+
372+
$start = hrtime(true);
373+
while ($indexesToCheck) {
374+
if (hrtime(true) > $start + $maxTimeMs * 1_000_000) {
375+
throw new MongoDBException(sprintf('Timed out waiting for search indexes to become queryable after %d ms. Search indexes are not ready for the following class(es): %s', $maxTimeMs, implode(', ', array_keys($indexesToCheck))));
376+
}
377+
378+
foreach ($indexesToCheck as $className => $indexNames) {
379+
$collection = $this->dm->getDocumentCollection($className);
380+
381+
/** @var array<string, bool> $indexStatus Queryable status for each index name */
382+
$indexStatus = array_column(iterator_to_array($collection->listSearchIndexes([
383+
'filter' => ['name' => ['$in' => array_keys($indexNames)]],
384+
'typeMap' => ['root' => 'array'],
385+
])), 'queryable', 'name');
386+
387+
// Check that all indexes exist
388+
$missingIndexes = array_diff_key($indexNames, array_keys($indexStatus));
389+
if ($missingIndexes) {
390+
throw SchemaException::missingSearchIndex($className, $missingIndexes);
391+
}
392+
393+
// Remove the indexes that are ready from the list of indexes to check
394+
$indexesToCheck[$className] = array_keys(array_filter($indexStatus, static fn ($queryable) => ! $queryable));
395+
}
396+
397+
// Remove empty arrays and wait before checking again
398+
($indexesToCheck = array_filter($indexesToCheck)) && usleep($waitTimeMs * 1_000);
399+
}
400+
}
401+
340402
/**
341403
* Create search indexes for the given document class.
342404
*
@@ -368,7 +430,7 @@ public function createDocumentSearchIndexes(string $documentName): void
368430
$unprocessedNames = array_diff($definedNames, $createdNames);
369431

370432
if (! empty($unprocessedNames)) {
371-
throw new InvalidArgumentException(sprintf('The following search indexes for %s were not created: %s', $class->name, implode(', ', $unprocessedNames)));
433+
throw SchemaException::missingSearchIndex($class->name, $unprocessedNames);
372434
}
373435
}
374436

‎phpstan-baseline.neon‎

Lines changed: 0 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -384,18 +384,6 @@ parameters:
384384
count: 1
385385
path: lib/Doctrine/ODM/MongoDB/Mapping/ClassMetadata.php
386386

387-
-
388-
message: '#^Class Doctrine\\ODM\\MongoDB\\Mapping\\ClassMetadata has type alias SearchIndexDefinition with no value type specified in iterable type array\.$#'
389-
identifier: missingType.iterableValue
390-
count: 2
391-
path: lib/Doctrine/ODM/MongoDB/Mapping/ClassMetadata.php
392-
393-
-
394-
message: '#^Class Doctrine\\ODM\\MongoDB\\Mapping\\ClassMetadata has type alias SearchIndexMapping with no value type specified in iterable type array\.$#'
395-
identifier: missingType.iterableValue
396-
count: 2
397-
path: lib/Doctrine/ODM/MongoDB/Mapping/ClassMetadata.php
398-
399387
-
400388
message: '#^Instanceof between Doctrine\\Persistence\\Reflection\\RuntimeReflectionProperty and ReflectionProperty will always evaluate to true\.$#'
401389
identifier: instanceof.alwaysTrue
@@ -408,24 +396,12 @@ parameters:
408396
count: 1
409397
path: lib/Doctrine/ODM/MongoDB/Mapping/ClassMetadata.php
410398

411-
-
412-
message: '#^Method Doctrine\\ODM\\MongoDB\\Mapping\\ClassMetadata\:\:addSearchIndex\(\) has parameter \$definition with no value type specified in iterable type array\.$#'
413-
identifier: missingType.iterableValue
414-
count: 2
415-
path: lib/Doctrine/ODM/MongoDB/Mapping/ClassMetadata.php
416-
417399
-
418400
message: '#^Method Doctrine\\ODM\\MongoDB\\Mapping\\ClassMetadata\:\:getBucketName\(\) never returns null so it can be removed from the return type\.$#'
419401
identifier: return.unusedType
420402
count: 1
421403
path: lib/Doctrine/ODM/MongoDB/Mapping/ClassMetadata.php
422404

423-
-
424-
message: '#^Method Doctrine\\ODM\\MongoDB\\Mapping\\ClassMetadata\:\:getSearchIndexes\(\) return type has no value type specified in iterable type array\.$#'
425-
identifier: missingType.iterableValue
426-
count: 2
427-
path: lib/Doctrine/ODM/MongoDB/Mapping/ClassMetadata.php
428-
429405
-
430406
message: '#^Method Doctrine\\ODM\\MongoDB\\Mapping\\ClassMetadata\:\:mapField\(\) should return array\{type\: string, fieldName\: string, name\: string, isCascadeRemove\: bool, isCascadePersist\: bool, isCascadeRefresh\: bool, isCascadeMerge\: bool, isCascadeDetach\: bool, \.\.\.\} but returns array\{type\?\: string, fieldName\?\: string, name\?\: string, strategy\?\: string, association\?\: int, id\?\: bool, isOwningSide\?\: bool, collectionClass\?\: class\-string, \.\.\.\}\.$#'
431407
identifier: return.type
@@ -456,12 +432,6 @@ parameters:
456432
count: 1
457433
path: lib/Doctrine/ODM/MongoDB/Mapping/ClassMetadata.php
458434

459-
-
460-
message: '#^Property Doctrine\\ODM\\MongoDB\\Mapping\\ClassMetadata\:\:\$searchIndexes type has no value type specified in iterable type array\.$#'
461-
identifier: missingType.iterableValue
462-
count: 2
463-
path: lib/Doctrine/ODM/MongoDB/Mapping/ClassMetadata.php
464-
465435
-
466436
message: '#^Template type T is declared as covariant, but occurs in invariant position in property Doctrine\\ODM\\MongoDB\\Mapping\\ClassMetadata\:\:\$name\.$#'
467437
identifier: generics.variance

‎tests/Doctrine/ODM/MongoDB/Tests/Functional/AtlasSearchTest.php‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,16 @@
77
use Doctrine\ODM\MongoDB\Tests\BaseTestCase;
88
use Documents\CmsArticle;
99
use Documents\CmsUser;
10+
use MongoDB\Driver\WriteConcern;
1011
use PHPUnit\Framework\Attributes\Group;
1112

12-
use function sleep;
13-
1413
#[Group('atlas')]
1514
class AtlasSearchTest extends BaseTestCase
1615
{
1716
public function testAtlasSearch(): void
1817
{
1918
$schemaManager = $this->dm->getSchemaManager();
2019
$schemaManager->createDocumentCollection(CmsArticle::class);
21-
$schemaManager->createDocumentSearchIndexes(CmsArticle::class);
2220

2321
$user = new CmsUser();
2422
$user->status = 'active';
@@ -45,10 +43,15 @@ public function testAtlasSearch(): void
4543
$this->dm->persist($article1);
4644
$this->dm->persist($article2);
4745
$this->dm->persist($article3);
48-
$this->dm->flush();
46+
47+
// Write with majority concern to ensure data is visible for search
48+
$this->dm->flush(['writeConcern' => new WriteConcern(WriteConcern::MAJORITY)]);
49+
50+
// Index must be created after data insertion, so the index status is not immediately "READY"
51+
$schemaManager->createDocumentSearchIndexes(CmsArticle::class);
4952

5053
// Wait for the search index to be ready (Atlas Local needs time to build the index)
51-
sleep(2);
54+
$schemaManager->waitForSearchIndexes([CmsArticle::class, CmsUser::class]);
5255

5356
$results = $this->dm->createAggregationBuilder(CmsArticle::class)
5457
->search()

‎tests/Doctrine/ODM/MongoDB/Tests/Functional/SchemaManagerWaitForSearchIndexesTest.php‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace Doctrine\ODM\MongoDB\Tests\Functional;
6+
7+
use Doctrine\ODM\MongoDB\MongoDBException;
8+
use Doctrine\ODM\MongoDB\SchemaException;
9+
use Doctrine\ODM\MongoDB\Tests\BaseTestCase;
10+
use Documents\CmsArticle;
11+
use MongoDB\Driver\BulkWrite;
12+
use PHPUnit\Framework\Attributes\Group;
13+
use PHPUnit\Framework\Attributes\TestWith;
14+
15+
use function hrtime;
16+
17+
#[Group('atlas')]
18+
class SchemaManagerWaitForSearchIndexesTest extends BaseTestCase
19+
{
20+
#[TestWith([0])]
21+
#[TestWith([50_000])]
22+
public function testWait(int $nbDocuments): void
23+
{
24+
$schemaManager = $this->dm->getSchemaManager();
25+
$collection = $this->dm->getDocumentCollection(CmsArticle::class);
26+
27+
$schemaManager->createDocumentCollection(CmsArticle::class);
28+
29+
if ($nbDocuments) {
30+
$bulk = new BulkWrite();
31+
for ($i = 0; $i < $nbDocuments; $i++) {
32+
$bulk->insert(['topic' => 'topic ' . $i, 'title' => 'title ' . $i, 'text' => 'text ' . $i]);
33+
}
34+
35+
$collection->getManager()->executeBulkWrite($collection->getNamespace(), $bulk);
36+
}
37+
38+
// The index must be created after data insertion, so the index status is not immediately "READY"
39+
$schemaManager->createDocumentSearchIndexes(CmsArticle::class);
40+
41+
$this->assertNotSame('READY', $collection->listSearchIndexes(['name' => 'search_articles'])->current()['status']);
42+
43+
$start = hrtime(true);
44+
$schemaManager->waitForSearchIndexes([CmsArticle::class]);
45+
$timeMs = (hrtime(true) - $start) / 1_000_000;
46+
47+
$this->assertSame($nbDocuments, $collection->aggregate([
48+
[
49+
'$searchMeta' => [
50+
'index' => 'search_articles',
51+
'exists' => ['path' => '_id'],
52+
'count' => ['type' => 'total'],
53+
],
54+
],
55+
])->toArray()[0]['count']['total'], 'All documents are indexed');
56+
57+
$this->assertSame('READY', $collection->listSearchIndexes(['name' => 'search_articles'])->current()['status'], 'Ready after ' . $timeMs . ' ms');
58+
}
59+
60+
public function testErrors(): void
61+
{
62+
$schemaManager = $this->dm->getSchemaManager();
63+
64+
// Search index missing
65+
try {
66+
$schemaManager->waitForSearchIndexes([CmsArticle::class]);
67+
$this->fail('Expected SchemaException not thrown');
68+
} catch (SchemaException $exception) {
69+
$this->assertSame('The document class "Documents\CmsArticle" is missing the following search index(es): "search_articles"', $exception->getMessage());
70+
}
71+
72+
$schemaManager->createDocumentCollection(CmsArticle::class);
73+
$schemaManager->createDocumentSearchIndexes(CmsArticle::class);
74+
75+
// Timeout too short
76+
try {
77+
$schemaManager->waitForSearchIndexes([CmsArticle::class], 1);
78+
$this->fail('Expected SchemaException not thrown');
79+
} catch (MongoDBException $exception) {
80+
$this->assertSame('Timed out waiting for search indexes to become queryable after 1 ms. Search indexes are not ready for the following class(es): Documents\CmsArticle', $exception->getMessage());
81+
}
82+
83+
// Not specifying classes waits for all
84+
try {
85+
$schemaManager->waitForSearchIndexes();
86+
$this->fail('Expected SchemaException not thrown');
87+
} catch (SchemaException $exception) {
88+
// The missing class varies depending on the test execution order,
89+
// classes are added to the ClassMetadataFactory in the order they are used
90+
$this->assertMatchesRegularExpression('#The document class "Documents\\\\(CmsAddress|VectorEmbedding)" is missing the following search index\(es\): "default"#', $exception->getMessage());
91+
}
92+
93+
// Remove the collection
94+
$schemaManager->dropDocumentCollection(CmsArticle::class);
95+
96+
try {
97+
$schemaManager->waitForSearchIndexes([CmsArticle::class]);
98+
$this->fail('Expected SchemaException not thrown');
99+
} catch (SchemaException $exception) {
100+
$this->assertSame('The document class "Documents\CmsArticle" is missing the following search index(es): "search_articles"', $exception->getMessage());
101+
}
102+
}
103+
}

‎tests/Doctrine/ODM/MongoDB/Tests/Functional/VectorSearchTest.php‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,9 @@
66

77
use Doctrine\ODM\MongoDB\Tests\BaseTestCase;
88
use Documents\VectorEmbedding;
9+
use MongoDB\Driver\WriteConcern;
910
use PHPUnit\Framework\Attributes\Group;
1011

11-
use function sleep;
12-
1312
#[Group('atlas')]
1413
class VectorSearchTest extends BaseTestCase
1514
{
@@ -20,7 +19,6 @@ public function testAtlasVectorSearch(): void
2019

2120
// Create the collection and vector search indexes
2221
$schemaManager->createDocumentCollection(VectorEmbedding::class);
23-
$schemaManager->createDocumentSearchIndexes(VectorEmbedding::class);
2422

2523
// Insert some test documents with vector embeddings
2624
$doc1 = new VectorEmbedding();
@@ -41,10 +39,14 @@ public function testAtlasVectorSearch(): void
4139
$this->dm->persist($doc1);
4240
$this->dm->persist($doc2);
4341
$this->dm->persist($doc3);
44-
$this->dm->flush();
42+
// Write with majority concern to ensure data is visible for search
43+
$this->dm->flush(['writeConcern' => new WriteConcern(WriteConcern::MAJORITY)]);
44+
45+
// Index must be created after data insertion, so the index status is not immediately "READY"
46+
$schemaManager->createDocumentSearchIndexes(VectorEmbedding::class);
4547

4648
// Wait for search index to be ready (Atlas Local needs time to build the index)
47-
sleep(2);
49+
$schemaManager->waitForSearchIndexes([VectorEmbedding::class]);
4850

4951
$results = $this->dm->createAggregationBuilder(VectorEmbedding::class)
5052
->vectorSearch()

‎tests/Doctrine/ODM/MongoDB/Tests/SchemaManagerTest.php‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
use Doctrine\ODM\MongoDB\DocumentManager;
1010
use Doctrine\ODM\MongoDB\Mapping\ClassMetadata;
1111
use Doctrine\ODM\MongoDB\Mapping\TimeSeries\Granularity;
12+
use Doctrine\ODM\MongoDB\SchemaException;
1213
use Doctrine\ODM\MongoDB\SchemaManager;
1314
use Documents\BaseDocument;
1415
use Documents\CmsAddress;
@@ -418,8 +419,8 @@ public function testCreateDocumentSearchIndexesNotCreatedError(): void
418419
->with($this->anything())
419420
->willReturn(['foo']);
420421

421-
$this->expectException(InvalidArgumentException::class);
422-
$this->expectExceptionMessage('The following search indexes for Documents\CmsArticle were not created: search_articles');
422+
$this->expectException(SchemaException::class);
423+
$this->expectExceptionMessage('The document class "Documents\CmsArticle" is missing the following search index(es): "search_articles"');
423424

424425
$this->schemaManager->createDocumentSearchIndexes(CmsArticle::class);
425426
}
@@ -486,9 +487,9 @@ public function testCreateVectorSearchIndex(): void
486487
'name' => 'vector_int',
487488
'definition' => [
488489
'fields' => [
489-
['type' => 'vector', 'path' => 'vectorInt', 'numDimensions' => 3, 'similarity' => 'cosine'],
490490
['type' => 'filter', 'path' => 'filterField'],
491491
['type' => 'filter', 'path' => 'not_mapped_filter'],
492+
['type' => 'vector', 'path' => 'vectorInt', 'numDimensions' => 3, 'similarity' => 'cosine'],
492493
],
493494
],
494495
],

0 commit comments

Comments
 (0)