Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introducing partial reindexes (BC Break) #460

Merged
merged 13 commits into from
Dec 18, 2024
3 changes: 2 additions & 1 deletion .examples/laravel/app/Search/BlogReindexProvider.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

namespace App\Search;

use CmsIg\Seal\Reindex\ReindexConfig;
use CmsIg\Seal\Reindex\ReindexProviderInterface;

class BlogReindexProvider implements ReindexProviderInterface
Expand All @@ -13,7 +14,7 @@ public function total(): int|null
return 3;
}

public function provide(): \Generator
public function provide(ReindexConfig $reindexConfig): \Generator
{
yield [
'id' => 1,
Expand Down
3 changes: 2 additions & 1 deletion .examples/mezzio/src/App/src/Search/BlogReindexProvider.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

namespace App\Search;

use CmsIg\Seal\Reindex\ReindexConfig;
use CmsIg\Seal\Reindex\ReindexProviderInterface;

class BlogReindexProvider implements ReindexProviderInterface
Expand All @@ -13,7 +14,7 @@ public function total(): int|null
return 3;
}

public function provide(): \Generator
public function provide(ReindexConfig $reindexConfig): \Generator
{
yield [
'id' => 1,
Expand Down
3 changes: 2 additions & 1 deletion .examples/spiral/app/src/Search/BlogReindexProvider.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

namespace App\Search;

use CmsIg\Seal\Reindex\ReindexConfig;
use CmsIg\Seal\Reindex\ReindexProviderInterface;

class BlogReindexProvider implements ReindexProviderInterface
Expand All @@ -13,7 +14,7 @@ public function total(): int|null
return 3;
}

public function provide(): \Generator
public function provide(ReindexConfig $partialReindexConfig): \Generator
{
yield [
'id' => 1,
Expand Down
3 changes: 2 additions & 1 deletion .examples/symfony/src/Search/BlogReindexProvider.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

namespace App\Search;

use CmsIg\Seal\Reindex\ReindexConfig;
use CmsIg\Seal\Reindex\ReindexProviderInterface;

class BlogReindexProvider implements ReindexProviderInterface
Expand All @@ -13,7 +14,7 @@ public function total(): int|null
return 3;
}

public function provide(): \Generator
public function provide(ReindexConfig $reindexConfig): \Generator
{
yield [
'id' => 1,
Expand Down
3 changes: 2 additions & 1 deletion .examples/yii/src/Search/BlogReindexProvider.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

namespace App\Search;

use CmsIg\Seal\Reindex\ReindexConfig;
use CmsIg\Seal\Reindex\ReindexProviderInterface;

class BlogReindexProvider implements ReindexProviderInterface
Expand All @@ -13,7 +14,7 @@ public function total(): int|null
return 3;
}

public function provide(): \Generator
public function provide(ReindexConfig $reindexConfig): \Generator
{
yield [
'id' => 1,
Expand Down
10 changes: 7 additions & 3 deletions integrations/laravel/src/Console/ReindexCommand.php
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
namespace CmsIg\Seal\Integration\Laravel\Console;

use CmsIg\Seal\EngineRegistry;
use CmsIg\Seal\Reindex\ReindexConfig;
use CmsIg\Seal\Reindex\ReindexProviderInterface;
use Illuminate\Console\Command;

Expand Down Expand Up @@ -60,6 +61,11 @@ public function handle(
/** @var int $bulkSize */
$bulkSize = ((int) $this->option('bulk-size')) ?: 100;

$reindexConfig = ReindexConfig::create()
->withIndex($indexName)
->withBulkSize($bulkSize)
->withDropIndex($drop);

foreach ($engineRegistry->getEngines() as $name => $engine) {
if ($engineName && $engineName !== $name) {
continue;
Expand All @@ -71,9 +77,7 @@ public function handle(

$engine->reindex(
$this->reindexProviders,
$indexName,
$drop,
$bulkSize,
$reindexConfig,
function (string $index, int $count, int|null $total) use ($progressBar) {
if (null !== $total) {
$progressBar->setMaxSteps($total);
Expand Down
10 changes: 7 additions & 3 deletions integrations/mezzio/src/Command/ReindexCommand.php
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
namespace CmsIg\Seal\Integration\Mezzio\Command;

use CmsIg\Seal\EngineRegistry;
use CmsIg\Seal\Reindex\ReindexConfig;
use CmsIg\Seal\Reindex\ReindexProviderInterface;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
Expand Down Expand Up @@ -57,6 +58,11 @@ protected function execute(InputInterface $input, OutputInterface $output): int
/** @var int $bulkSize */
$bulkSize = ((int) $input->getOption('bulk-size')) ?: 100; // @phpstan-ignore-line

$reindexConfig = ReindexConfig::create()
->withIndex($indexName)
->withBulkSize($bulkSize)
->withDropIndex($drop);

foreach ($this->engineRegistry->getEngines() as $name => $engine) {
if ($engineName && $engineName !== $name) {
continue;
Expand All @@ -68,9 +74,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int

$engine->reindex(
$this->reindexProviders,
$indexName,
$drop,
$bulkSize,
$reindexConfig,
function (string $index, int $count, int|null $total) use ($progressBar) {
if (null !== $total) {
$progressBar->setMaxSteps($total);
Expand Down
10 changes: 7 additions & 3 deletions integrations/spiral/src/Console/ReindexCommand.php
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
namespace CmsIg\Seal\Integration\Spiral\Console;

use CmsIg\Seal\EngineRegistry;
use CmsIg\Seal\Reindex\ReindexConfig;
use CmsIg\Seal\Reindex\ReindexProviderInterface;
use Spiral\Console\Attribute\AsCommand;
use Spiral\Console\Attribute\Option;
Expand Down Expand Up @@ -53,6 +54,11 @@ public function __construct(
public function __invoke(
EngineRegistry $engineRegistry,
): int {
$reindexConfig = ReindexConfig::create()
->withIndex($this->indexName)
->withBulkSize($this->bulkSize)
->withDropIndex($this->drop);

foreach ($engineRegistry->getEngines() as $name => $engine) {
if ($this->engineName && $this->engineName !== $name) {
continue;
Expand All @@ -64,9 +70,7 @@ public function __invoke(

$engine->reindex(
$this->reindexProviders,
$this->indexName,
$this->drop,
$this->bulkSize,
$reindexConfig,
function (string $index, int $count, int|null $total) use ($progressBar) {
if (null !== $total) {
$progressBar->setMaxSteps($total);
Expand Down
22 changes: 17 additions & 5 deletions integrations/symfony/src/Command/ReindexCommand.php
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
namespace CmsIg\Seal\Integration\Symfony\Command;

use CmsIg\Seal\EngineRegistry;
use CmsIg\Seal\Reindex\ReindexConfig;
use CmsIg\Seal\Reindex\ReindexProviderInterface;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
Expand Down Expand Up @@ -43,7 +44,9 @@ protected function configure(): void
$this->addOption('engine', null, InputOption::VALUE_REQUIRED, 'The name of the engine to create the schema for.');
$this->addOption('index', null, InputOption::VALUE_REQUIRED, 'The name of the index to create the schema for.');
$this->addOption('drop', null, InputOption::VALUE_NONE, 'Drop the index before reindexing.');
$this->addOption('bulk-size', null, InputOption::VALUE_REQUIRED, 'The bulk size for reindexing, defaults to 100.');
$this->addOption('bulk-size', null, InputOption::VALUE_REQUIRED, 'The bulk size for reindexing, defaults to 100.', 100);
$this->addOption('datetime-boundary', null, InputOption::VALUE_REQUIRED, 'Do a partial update and limit to only documents that have been changed since a given datetime object.');
$this->addOption('identifiers', null, InputOption::VALUE_REQUIRED, 'Do a partial update and limit to only a comma-separated list of identifiers.');
}

protected function execute(InputInterface $input, OutputInterface $output): int
Expand All @@ -56,7 +59,18 @@ protected function execute(InputInterface $input, OutputInterface $output): int
/** @var bool $drop */
$drop = $input->getOption('drop');
/** @var int $bulkSize */
$bulkSize = ((int) $input->getOption('bulk-size')) ?: 100; // @phpstan-ignore-line
$bulkSize = $input->getOption('bulk-size');
/** @var \DateTimeImmutable|null $dateTimeBoundary */
$dateTimeBoundary = $input->getOption('datetime-boundary') ? new \DateTimeImmutable((string) $input->getOption('datetime-boundary')) : null; // @phpstan-ignore-line
/** @var array<string> $identifiers */
$identifiers = \explode(',', (string) $input->getOption('identifiers')); // @phpstan-ignore-line

$reindexConfig = ReindexConfig::create()
->withIndex($indexName)
->withBulkSize($bulkSize)
->withDropIndex($drop)
->withDateTimeBoundary($dateTimeBoundary)
->withIdentifiers($identifiers);

foreach ($this->engineRegistry->getEngines() as $name => $engine) {
if ($engineName && $engineName !== $name) {
Expand All @@ -69,9 +83,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int

$engine->reindex(
$this->reindexProviders,
$indexName,
$drop,
$bulkSize,
$reindexConfig,
function (string $index, int $count, int|null $total) use ($progressBar) {
if (null !== $total) {
$progressBar->setMaxSteps($total);
Expand Down
10 changes: 7 additions & 3 deletions integrations/yii/src/Command/ReindexCommand.php
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
namespace CmsIg\Seal\Integration\Yii\Command;

use CmsIg\Seal\EngineRegistry;
use CmsIg\Seal\Reindex\ReindexConfig;
use CmsIg\Seal\Reindex\ReindexProviderInterface;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
Expand Down Expand Up @@ -57,6 +58,11 @@ protected function execute(InputInterface $input, OutputInterface $output): int
/** @var int $bulkSize */
$bulkSize = ((int) $input->getOption('bulk-size')) ?: 100; // @phpstan-ignore-line

$reindexConfig = ReindexConfig::create()
->withIndex($indexName)
->withBulkSize($bulkSize)
->withDropIndex($drop);

foreach ($this->engineRegistry->getEngines() as $name => $engine) {
if ($engineName && $engineName !== $name) {
continue;
Expand All @@ -68,9 +74,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int

$engine->reindex(
$this->reindexProviders,
$indexName,
$drop,
$bulkSize,
$reindexConfig,
function (string $index, int $count, int|null $total) use ($progressBar) {
if (null !== $total) {
$progressBar->setMaxSteps($total);
Expand Down
17 changes: 8 additions & 9 deletions packages/seal/src/Engine.php
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

use CmsIg\Seal\Adapter\AdapterInterface;
use CmsIg\Seal\Exception\DocumentNotFoundException;
use CmsIg\Seal\Reindex\ReindexConfig;
use CmsIg\Seal\Reindex\ReindexProviderInterface;
use CmsIg\Seal\Schema\Schema;
use CmsIg\Seal\Search\Condition\IdentifierCondition;
Expand Down Expand Up @@ -134,9 +135,7 @@ public function dropSchema(array $options = []): TaskInterface|null

public function reindex(
iterable $reindexProviders,
string|null $index = null,
bool $dropIndex = false,
int $bulkSize = 100,
ReindexConfig $reindexConfig,
callable|null $progressCallback = null,
): void {
/** @var array<string, ReindexProviderInterface[]> $reindexProvidersPerIndex */
Expand All @@ -146,13 +145,13 @@ public function reindex(
continue;
}

if ($reindexProvider::getIndex() === $index || null === $index) {
if ($reindexProvider::getIndex() === $reindexConfig->getIndex() || null === $reindexConfig->getIndex()) {
$reindexProvidersPerIndex[$reindexProvider::getIndex()][] = $reindexProvider;
}
}

foreach ($reindexProvidersPerIndex as $index => $reindexProviders) {
if ($dropIndex && $this->existIndex($index)) {
if ($reindexConfig->shouldDropIndex() && $this->existIndex($index)) {
$task = $this->dropIndex($index, ['return_slow_promise_result' => true]);
$task->wait();
$task = $this->createIndex($index, ['return_slow_promise_result' => true]);
Expand All @@ -165,18 +164,18 @@ public function reindex(
foreach ($reindexProviders as $reindexProvider) {
$this->bulk(
$index,
(function () use ($index, $reindexProvider, $bulkSize, $progressCallback) {
(function () use ($index, $reindexProvider, $reindexConfig, $progressCallback) {
$count = 0;
$total = $reindexProvider->total();

$lastCount = -1;
foreach ($reindexProvider->provide() as $document) {
foreach ($reindexProvider->provide($reindexConfig) as $document) {
++$count;

yield $document;

if (null !== $progressCallback
&& 0 === ($count % $bulkSize)
&& 0 === ($count % $reindexConfig->getBulkSize())
) {
$lastCount = $count;
$progressCallback($index, $count, $total);
Expand All @@ -190,7 +189,7 @@ public function reindex(
}
})(),
[],
$bulkSize,
$reindexConfig->getBulkSize(),
);
}
}
Expand Down
5 changes: 2 additions & 3 deletions packages/seal/src/EngineInterface.php
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
namespace CmsIg\Seal;

use CmsIg\Seal\Exception\DocumentNotFoundException;
use CmsIg\Seal\Reindex\ReindexConfig;
use CmsIg\Seal\Reindex\ReindexProviderInterface;
use CmsIg\Seal\Search\SearchBuilder;
use CmsIg\Seal\Task\TaskInterface;
Expand Down Expand Up @@ -92,9 +93,7 @@ public function dropSchema(array $options = []): TaskInterface|null;
*/
public function reindex(
iterable $reindexProviders,
string|null $index = null,
bool $dropIndex = false,
int $bulkSize = 100,
ReindexConfig $reindexConfig,
callable|null $progressCallback = null,
): void;
}
Loading
Loading