From a0ba2df3b5199d02e5a6c97dc674d82d989d726b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joachim=20L=C3=B8vgaard?= Date: Wed, 21 Aug 2024 12:31:59 +0200 Subject: [PATCH] Add data provider for indexes --- src/Config/Index.php | 28 +++++++---- .../DefaultIndexableDataProvider.php | 41 ++++++++++++++++ .../IndexableDataProviderInterface.php | 20 ++++++++ src/DependencyInjection/Configuration.php | 15 ++++-- .../SetonoSyliusMeilisearchExtension.php | 11 +++-- ...sedQueryBuilderForIndexingCreatedEvent.php | 18 ------- .../QueryBuilderForDataProvisionCreated.php | 20 ++++++++ src/Indexer/DefaultIndexer.php | 41 ++++++---------- src/Indexer/IndexBuffer.php | 47 +++++++++++++++++++ src/Message/Command/IndexEntities.php | 34 +++++++++++++- src/Model/IndexableInterface.php | 2 +- src/Resources/config/services.xml | 1 + .../config/services/data_provider.xml | 13 +++++ 13 files changed, 228 insertions(+), 63 deletions(-) create mode 100644 src/DataProvider/DefaultIndexableDataProvider.php create mode 100644 src/DataProvider/IndexableDataProviderInterface.php delete mode 100644 src/Event/EntityBasedQueryBuilderForIndexingCreatedEvent.php create mode 100644 src/Event/QueryBuilderForDataProvisionCreated.php create mode 100644 src/Indexer/IndexBuffer.php create mode 100644 src/Resources/config/services/data_provider.xml diff --git a/src/Config/Index.php b/src/Config/Index.php index adf2891..23bcc50 100644 --- a/src/Config/Index.php +++ b/src/Config/Index.php @@ -5,6 +5,7 @@ namespace Setono\SyliusMeilisearchPlugin\Config; use Psr\Container\ContainerInterface; +use Setono\SyliusMeilisearchPlugin\DataProvider\IndexableDataProviderInterface; use Setono\SyliusMeilisearchPlugin\Document\Document; use Setono\SyliusMeilisearchPlugin\Indexer\IndexerInterface; use Setono\SyliusMeilisearchPlugin\Model\IndexableInterface; @@ -53,15 +54,6 @@ public function __construct( } } - /** - * @psalm-suppress MixedInferredReturnType - */ - public function indexer(): IndexerInterface - { - /** @psalm-suppress MixedReturnStatement */ - return $this->locator->get(IndexerInterface::class); - } - /** * @param class-string|object $entity */ @@ -80,6 +72,24 @@ public function hasEntity(string|object $entity): bool return false; } + /** + * @psalm-suppress MixedInferredReturnType + */ + public function indexer(): IndexerInterface + { + /** @psalm-suppress MixedReturnStatement */ + return $this->locator->get(IndexerInterface::class); + } + + /** + * @psalm-suppress MixedInferredReturnType + */ + public function dataProvider(): IndexableDataProviderInterface + { + /** @psalm-suppress MixedReturnStatement */ + return $this->locator->get(IndexableDataProviderInterface::class); + } + public function __toString(): string { return $this->name; diff --git a/src/DataProvider/DefaultIndexableDataProvider.php b/src/DataProvider/DefaultIndexableDataProvider.php new file mode 100644 index 0000000..fa09290 --- /dev/null +++ b/src/DataProvider/DefaultIndexableDataProvider.php @@ -0,0 +1,41 @@ +managerRegistry = $managerRegistry; + } + + public function getIds(string $entity, Index $index): \Generator + { + $qb = $this + ->getManager($entity) + ->createQueryBuilder() + ->select('o.id') + ->from($entity, 'o') + ; + + $this->eventDispatcher->dispatch(new QueryBuilderForDataProvisionCreated($entity, $index, $qb)); + + /** @var SelectBatchIteratorAggregate $batch */ + $batch = SelectBatchIteratorAggregate::fromQuery($qb->getQuery(), 100); + + yield from $batch; + } +} diff --git a/src/DataProvider/IndexableDataProviderInterface.php b/src/DataProvider/IndexableDataProviderInterface.php new file mode 100644 index 0000000..7a62035 --- /dev/null +++ b/src/DataProvider/IndexableDataProviderInterface.php @@ -0,0 +1,20 @@ + $entity + * + * @return iterable + */ + public function getIds(string $entity, Index $index): iterable; +} diff --git a/src/DependencyInjection/Configuration.php b/src/DependencyInjection/Configuration.php index ebc3630..0a285ff 100644 --- a/src/DependencyInjection/Configuration.php +++ b/src/DependencyInjection/Configuration.php @@ -4,6 +4,7 @@ namespace Setono\SyliusMeilisearchPlugin\DependencyInjection; +use Setono\SyliusMeilisearchPlugin\DataProvider\DefaultIndexableDataProvider; use Setono\SyliusMeilisearchPlugin\Document\Product; use Setono\SyliusMeilisearchPlugin\Form\Type\SynonymType; use Setono\SyliusMeilisearchPlugin\Indexer\DefaultIndexer; @@ -36,20 +37,26 @@ public function getConfigTreeBuilder(): TreeBuilder ->beforeNormalization()->castToArray()->end() ->defaultValue([]) ->arrayPrototype() + ->addDefaultsIfNotSet() ->children() ->scalarNode('document') ->info(sprintf('The fully qualified class name for the document that maps to the index. If you are creating a product index, a good starting point is the %s', Product::class)) ->cannotBeEmpty() ->isRequired() ->end() - ->scalarNode('indexer') - ->info(sprintf('You can set a custom indexer here. If you do not set one, the default indexer will be used. The default indexer is %s', DefaultIndexer::class)) - ->defaultNull() - ->end() ->arrayNode('entities') ->info('The Doctrine entities that make up this index. Examples could be "App\Entity\Product\Product", "App\Entity\Taxonomy\Taxon", etc.') ->scalarPrototype()->end() ->end() + ->scalarNode('data_provider') + ->info('You can set a custom data provider here. If you do not set one, the default data provider will be used.') + ->defaultValue(DefaultIndexableDataProvider::class) + ->end() + ->scalarNode('indexer') + ->info(sprintf('You can set a custom indexer here. If you do not set one, the default indexer will be used. The default indexer is %s', DefaultIndexer::class)) + ->cannotBeEmpty() + ->defaultNull() + ->end() ->scalarNode('prefix') ->defaultNull() ->info('If you want to prepend a string to the index name, you can set it here. This can be useful in a development setup where each developer has their own prefix. Notice that the environment is already prefixed by default, so you do not have to prefix that.') diff --git a/src/DependencyInjection/SetonoSyliusMeilisearchExtension.php b/src/DependencyInjection/SetonoSyliusMeilisearchExtension.php index 5b4a2a7..29a9d14 100644 --- a/src/DependencyInjection/SetonoSyliusMeilisearchExtension.php +++ b/src/DependencyInjection/SetonoSyliusMeilisearchExtension.php @@ -7,6 +7,7 @@ use Meilisearch\Client; use Setono\SyliusMeilisearchPlugin\Config\Index; use Setono\SyliusMeilisearchPlugin\DataMapper\DataMapperInterface; +use Setono\SyliusMeilisearchPlugin\DataProvider\IndexableDataProviderInterface; use Setono\SyliusMeilisearchPlugin\Document\Document; use Setono\SyliusMeilisearchPlugin\Filter\Doctrine\FilterInterface as DoctrineFilterInterface; use Setono\SyliusMeilisearchPlugin\Filter\Object\FilterInterface as ObjectFilterInterface; @@ -34,7 +35,7 @@ public function load(array $configs, ContainerBuilder $container): void * @psalm-suppress PossiblyNullArgument * * @var array{ - * indexes: array, indexer: string|null, entities: list, prefix: string|null}>, + * indexes: array, entities: list, data_provider: class-string, indexer: class-string|null, prefix: string|null}>, * server: array{ host: string, master_key: string }, * search: array{ enabled: bool, path: string, index: string, hits_per_page: integer }, * resources: array, @@ -159,7 +160,7 @@ public function prepend(ContainerBuilder $container): void } /** - * @param array, indexer: string|null, entities: list, prefix: string|null}> $config + * @param array, entities: list, data_provider: class-string, indexer: class-string|null, prefix: string|null}> $config */ private static function registerIndexesConfiguration(array $config, ContainerBuilder $container): void { @@ -178,7 +179,10 @@ private static function registerIndexesConfiguration(array $config, ContainerBui $indexName, $index['document'], $index['entities'], - ServiceLocatorTagPass::register($container, [IndexerInterface::class => new Reference($indexerServiceId)]), + ServiceLocatorTagPass::register($container, [ + IndexableDataProviderInterface::class => new Reference($index['data_provider']), + IndexerInterface::class => new Reference($indexerServiceId), + ]), $index['prefix'], ])); @@ -203,6 +207,7 @@ private static function registerDefaultIndexer(ContainerBuilder $container, stri new Reference(Client::class), new Reference('setono_sylius_meilisearch.filter.object.composite'), new Reference('event_dispatcher'), + new Reference('setono_sylius_meilisearch.command_bus'), ])); return $indexerServiceId; diff --git a/src/Event/EntityBasedQueryBuilderForIndexingCreatedEvent.php b/src/Event/EntityBasedQueryBuilderForIndexingCreatedEvent.php deleted file mode 100644 index 4b22ab8..0000000 --- a/src/Event/EntityBasedQueryBuilderForIndexingCreatedEvent.php +++ /dev/null @@ -1,18 +0,0 @@ - $entityClass */ - public readonly string $entityClass, - public readonly QueryBuilder $queryBuilder, - ) { - } -} diff --git a/src/Event/QueryBuilderForDataProvisionCreated.php b/src/Event/QueryBuilderForDataProvisionCreated.php new file mode 100644 index 0000000..7ad1c1e --- /dev/null +++ b/src/Event/QueryBuilderForDataProvisionCreated.php @@ -0,0 +1,20 @@ + $entity */ + public readonly string $entity, + public readonly Index $index, + public readonly QueryBuilder $qb, + ) { + } +} diff --git a/src/Indexer/DefaultIndexer.php b/src/Indexer/DefaultIndexer.php index d091dd9..696d807 100644 --- a/src/Indexer/DefaultIndexer.php +++ b/src/Indexer/DefaultIndexer.php @@ -5,18 +5,17 @@ namespace Setono\SyliusMeilisearchPlugin\Indexer; use Doctrine\Persistence\ManagerRegistry; -use DoctrineBatchUtils\BatchProcessing\SelectBatchIteratorAggregate; use Meilisearch\Client; use Psr\EventDispatcher\EventDispatcherInterface; use Setono\Doctrine\ORMTrait; use Setono\SyliusMeilisearchPlugin\Config\Index; use Setono\SyliusMeilisearchPlugin\DataMapper\DataMapperInterface; use Setono\SyliusMeilisearchPlugin\Document\Document; -use Setono\SyliusMeilisearchPlugin\Event\EntityBasedQueryBuilderForIndexingCreatedEvent; use Setono\SyliusMeilisearchPlugin\Filter\Object\FilterInterface as ObjectFilterInterface; -use Setono\SyliusMeilisearchPlugin\Model\IndexableInterface; +use Setono\SyliusMeilisearchPlugin\Message\Command\IndexEntities; use Setono\SyliusMeilisearchPlugin\Provider\IndexScope\IndexScopeProviderInterface; use Setono\SyliusMeilisearchPlugin\Resolver\IndexName\IndexNameResolverInterface; +use Symfony\Component\Messenger\MessageBusInterface; use Symfony\Component\Serializer\Normalizer\NormalizerInterface; use Webmozart\Assert\Assert; @@ -37,6 +36,7 @@ public function __construct( protected readonly Client $client, protected readonly ObjectFilterInterface $objectFilter, protected readonly EventDispatcherInterface $eventDispatcher, + protected readonly MessageBusInterface $commandBus, ) { $this->managerRegistry = $managerRegistry; } @@ -44,7 +44,14 @@ public function __construct( public function index(): void { foreach ($this->index->entities as $entity) { - $this->indexEntityClass($entity); + /** @var IndexBuffer $buffer */ + $buffer = new IndexBuffer(100, fn (array $ids) => $this->commandBus->dispatch(IndexEntities::fromIds($entity, $ids))); + + foreach ($this->index->dataProvider()->getIds($entity, $this->index) as $id) { + $buffer->push($id); + } + + $buffer->flush(); } } @@ -61,7 +68,9 @@ public function indexEntities(array $entities): void $document = new $this->index->document(); $this->dataMapper->map($entity, $document, $indexScope); - $this->objectFilter->filter($entity, $document, $indexScope); + if (!$this->objectFilter->filter($entity, $document, $indexScope)) { + continue; + } $documents[] = $this->normalize($document); } @@ -83,28 +92,6 @@ public function removeEntities(array $entities): void } } - /** - * @param class-string $entity - */ - protected function indexEntityClass(string $entity): void - { - $qb = $this - ->getManager($entity) - ->createQueryBuilder() - ->select('o') - ->from($entity, 'o') - ; - - $this->eventDispatcher->dispatch(new EntityBasedQueryBuilderForIndexingCreatedEvent($entity, $qb)); - - /** @var SelectBatchIteratorAggregate $objects */ - $objects = SelectBatchIteratorAggregate::fromQuery($qb->getQuery(), 100); - - foreach ($objects as $object) { - $this->indexEntity($object); - } - } - // todo move this to a service protected function normalize(Document $document): array { diff --git a/src/Indexer/IndexBuffer.php b/src/Indexer/IndexBuffer.php new file mode 100644 index 0000000..16a4ab5 --- /dev/null +++ b/src/Indexer/IndexBuffer.php @@ -0,0 +1,47 @@ + */ + private array $buffer = []; + + /** + * @param \Closure(list):void $callback + */ + public function __construct(private readonly int $bufferSize, private readonly \Closure $callback) + { + } + + /** + * @param T $item + */ + public function push(mixed $item): void + { + $this->buffer[] = $item; + ++$this->count; + + if ($this->count >= $this->bufferSize) { + $this->flush(); + } + } + + public function flush(): void + { + if ($this->count > 0) { + ($this->callback)($this->buffer); + $this->buffer = []; + $this->count = 0; + } + } +} diff --git a/src/Message/Command/IndexEntities.php b/src/Message/Command/IndexEntities.php index bdf41f0..4d0d42a 100644 --- a/src/Message/Command/IndexEntities.php +++ b/src/Message/Command/IndexEntities.php @@ -9,12 +9,44 @@ final class IndexEntities implements CommandInterface { - public function __construct( + private function __construct( /** @var class-string $class */ public readonly string $class, + /** @var list $ids */ public readonly array $ids, ) { Assert::stringNotEmpty($class); Assert::notEmpty($ids); } + + /** + * @param list $entities + */ + public static function fromEntities(array $entities): self + { + $type = null; + $ids = []; + foreach ($entities as $entity) { + if (null === $type) { + $type = $entity::class; + } + + Assert::same($type, $entity::class, 'All entities must be of the same type'); + + $ids[] = $entity->getId(); + } + + Assert::notNull($type); + + return new self($type, $ids); + } + + /** + * @param class-string $class + * @param list $ids + */ + public static function fromIds(string $class, array $ids): self + { + return new self($class, $ids); + } } diff --git a/src/Model/IndexableInterface.php b/src/Model/IndexableInterface.php index ff8f6ed..455895b 100644 --- a/src/Model/IndexableInterface.php +++ b/src/Model/IndexableInterface.php @@ -9,7 +9,7 @@ interface IndexableInterface /** * This is compatible with Sylius' getId() method * - * @return mixed + * @return int|string|null */ public function getId(); diff --git a/src/Resources/config/services.xml b/src/Resources/config/services.xml index cd62f00..92cd05c 100644 --- a/src/Resources/config/services.xml +++ b/src/Resources/config/services.xml @@ -6,6 +6,7 @@ + diff --git a/src/Resources/config/services/data_provider.xml b/src/Resources/config/services/data_provider.xml new file mode 100644 index 0000000..ed6a1a4 --- /dev/null +++ b/src/Resources/config/services/data_provider.xml @@ -0,0 +1,13 @@ + + + + + + + + + + +