diff --git a/src/Command/IndexCommand.php b/src/Command/IndexCommand.php index ff0c7778..8c055ca6 100644 --- a/src/Command/IndexCommand.php +++ b/src/Command/IndexCommand.php @@ -36,6 +36,11 @@ protected function getIndices(): Collection }); } + protected function getIndexNameWithoutPrefix(string $prefixedIndexName): string + { + return preg_replace(\sprintf('/^%s/', preg_quote($this->prefix)), '', $prefixedIndexName) ?? $prefixedIndexName; + } + protected function getEntitiesFromArgs(InputInterface $input, OutputInterface $output): Collection { $indices = $this->getIndices(); diff --git a/src/Command/MeilisearchImportCommand.php b/src/Command/MeilisearchImportCommand.php index 30f8d342..a32d3578 100644 --- a/src/Command/MeilisearchImportCommand.php +++ b/src/Command/MeilisearchImportCommand.php @@ -6,6 +6,7 @@ use Doctrine\Persistence\ManagerRegistry; use Meilisearch\Bundle\Collection; +use Meilisearch\Bundle\DataProvider\DoctrineOrmDataProvider; use Meilisearch\Bundle\EventListener\ConsoleOutputSubscriber; use Meilisearch\Bundle\Exception\TaskException; use Meilisearch\Bundle\Model\Aggregator; @@ -99,10 +100,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int $totalIndexed = 0; $manager = $this->managerRegistry->getManagerForClass($entityClassName); - $repository = $manager->getRepository($entityClassName); - $classMetadata = $manager->getClassMetadata($entityClassName); - $entityIdentifiers = $classMetadata->getIdentifierFieldNames(); - $sortByAttrs = array_combine($entityIdentifiers, array_fill(0, \count($entityIdentifiers), 'ASC')); $output->writeln('Importing for index '.$entityClassName.''); @@ -119,12 +116,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int } do { - $entities = $repository->findBy( - [], - $sortByAttrs, - $batchSize, - $batchSize * $page - ); + $entities = $this->getEntities($index['name'], $entityClassName, $batchSize, $page); $responses = $this->formatIndexingResponse($this->searchService->index($manager, $entities), $responseTimeout); $totalIndexed += \count($entities); @@ -208,4 +200,24 @@ private function entitiesToIndex($indexes): array return array_unique($indexes->all(), SORT_REGULAR); } + + /** + * @param string $prefixedIndexName + * @param string $entityClassName + * @param int $batchSize + * @param int $page + * + * @return array + */ + private function getEntities($prefixedIndexName, $entityClassName, $batchSize, $page): array + { + $dataProvider = $this->searchService->getDataProvider($this->getIndexNameWithoutPrefix($prefixedIndexName)); + + if (null === $dataProvider) { + $dataProvider = new DoctrineOrmDataProvider($this->managerRegistry); + $dataProvider->setEntityClassName($entityClassName); + } + + return $dataProvider->getAll($batchSize, $batchSize * $page); + } } diff --git a/src/DataProvider/DataProvider.php b/src/DataProvider/DataProvider.php new file mode 100644 index 00000000..7f5228bc --- /dev/null +++ b/src/DataProvider/DataProvider.php @@ -0,0 +1,18 @@ +managerRegistry = $managerRegistry; + } + + public function setEntityClassName(string $entityClassName): void + { + $this->entityClassName = $entityClassName; + } + + public function getAll(int $limit = 100, int $offset = 0): array + { + if (empty($this->entityClassName)) { + throw new \Exception('No entity class name set on data provider.'); + } + + $manager = $this->managerRegistry->getManagerForClass($this->entityClassName); + $classMetadata = $manager->getClassMetadata($this->entityClassName); + $entityIdentifiers = $classMetadata->getIdentifierFieldNames(); + $repository = $manager->getRepository($this->entityClassName); + $sortByAttrs = array_combine($entityIdentifiers, array_fill(0, \count($entityIdentifiers), 'ASC')); + + return $repository->findBy( + [], + $sortByAttrs, + $limit, + $offset + ); + } +} diff --git a/src/DependencyInjection/Configuration.php b/src/DependencyInjection/Configuration.php index c3edcff7..af98287e 100644 --- a/src/DependencyInjection/Configuration.php +++ b/src/DependencyInjection/Configuration.php @@ -59,6 +59,10 @@ public function getConfigTreeBuilder(): TreeBuilder ->info('Property accessor path (like method or property name) used to decide if an entry should be indexed.') ->defaultNull() ->end() + ->scalarNode('data_provider') + ->info('Method of the entity repository called when the meilisearch:import command is invoked.') + ->defaultNull() + ->end() ->arrayNode('settings') ->info('Configure indices settings, see: https://www.meilisearch.com/docs/reference/api/settings') ->beforeNormalization() diff --git a/src/DependencyInjection/MeilisearchExtension.php b/src/DependencyInjection/MeilisearchExtension.php index e6692bb5..ce640473 100644 --- a/src/DependencyInjection/MeilisearchExtension.php +++ b/src/DependencyInjection/MeilisearchExtension.php @@ -30,6 +30,10 @@ public function load(array $configs, ContainerBuilder $container): void foreach ($config['indices'] as $index => $indice) { $config['indices'][$index]['prefixed_name'] = $config['prefix'].$indice['name']; $config['indices'][$index]['settings'] = $this->findReferences($config['indices'][$index]['settings']); + + if (null !== $config['indices'][$index]['data_provider']) { + $config['indices'][$index]['data_provider'] = new Reference($config['indices'][$index]['data_provider']); + } } $container->setParameter('meili_url', $config['url'] ?? null); diff --git a/src/SearchService.php b/src/SearchService.php index 9fbdcfef..e77d027e 100644 --- a/src/SearchService.php +++ b/src/SearchService.php @@ -5,6 +5,7 @@ namespace Meilisearch\Bundle; use Doctrine\Persistence\ObjectManager; +use Meilisearch\Bundle\DataProvider\DataProvider; interface SearchService { @@ -79,4 +80,6 @@ public function rawSearch( * @return int<0, max> */ public function count(string $className, string $query = '', array $searchParams = []): int; + + public function getDataProvider(string $indexName): ?DataProvider; } diff --git a/src/Services/MeilisearchService.php b/src/Services/MeilisearchService.php index a001fee2..0efd8655 100644 --- a/src/Services/MeilisearchService.php +++ b/src/Services/MeilisearchService.php @@ -8,6 +8,7 @@ use Doctrine\ORM\Proxy\DefaultProxyClassNameResolver; use Doctrine\Persistence\ObjectManager; use Meilisearch\Bundle\Collection; +use Meilisearch\Bundle\DataProvider\DataProvider; use Meilisearch\Bundle\Engine; use Meilisearch\Bundle\Entity\Aggregator; use Meilisearch\Bundle\Exception\ObjectIdNotFoundException; @@ -42,6 +43,7 @@ final class MeilisearchService implements SearchService */ private array $classToSerializerGroup; private array $indexIfMapping; + private array $dataProviderMapping; public function __construct(NormalizerInterface $normalizer, Engine $engine, array $configuration, ?PropertyAccessorInterface $propertyAccessor = null) { @@ -54,6 +56,7 @@ public function __construct(NormalizerInterface $normalizer, Engine $engine, arr $this->setAggregatorsAndEntitiesAggregators(); $this->setClassToSerializerGroup(); $this->setIndexIfMapping(); + $this->setDataProviderMapping(); } public function isSearchable($className): bool @@ -223,6 +226,11 @@ public function shouldBeIndexed(object $entity): bool return true; } + public function getDataProvider(string $indexName): ?DataProvider + { + return $this->dataProviderMapping[$indexName] ?? null; + } + /** * @param object|class-string $objectOrClass * @@ -295,6 +303,17 @@ private function setIndexIfMapping(): void $this->indexIfMapping = $mapping; } + private function setDataProviderMapping(): void + { + $mapping = []; + + /** @var array $indexDetails */ + foreach ($this->configuration->get('indices') as $indexDetails) { + $mapping[$indexDetails['name']] = $indexDetails['data_provider']; + } + $this->dataProviderMapping = $mapping; + } + /** * Returns the aggregators instances of the provided entities. * diff --git a/tests/BaseKernelTestCase.php b/tests/BaseKernelTestCase.php index 8137a37b..6704ea20 100644 --- a/tests/BaseKernelTestCase.php +++ b/tests/BaseKernelTestCase.php @@ -18,6 +18,7 @@ use Meilisearch\Bundle\Tests\Entity\Podcast; use Meilisearch\Bundle\Tests\Entity\Post; use Meilisearch\Bundle\Tests\Entity\Tag; +use Meilisearch\Bundle\Tests\Entity\Ticket; use Meilisearch\Client; use Meilisearch\Exceptions\ApiException; use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase; @@ -194,6 +195,16 @@ protected function createLink(array $properties = []): Link return $link; } + protected function createTicket(int $id, bool $sold): Ticket + { + $ticket = new Ticket($id, str_pad((string) random_int(0, 1000000), 6, '0', STR_PAD_LEFT), $sold); + + $this->entityManager->persist($ticket); + $this->entityManager->flush(); + + return $ticket; + } + protected function getPrefix(): string { return $this->searchService->getConfiguration()->get('prefix'); diff --git a/tests/DataProvider/TicketDataProvider.php b/tests/DataProvider/TicketDataProvider.php new file mode 100644 index 00000000..d76b5b5e --- /dev/null +++ b/tests/DataProvider/TicketDataProvider.php @@ -0,0 +1,32 @@ +managerRegistry = $managerRegistry; + } + + public function getAll(int $limit = 100, int $offset = 0): array + { + $manager = $this->managerRegistry->getManagerForClass(Ticket::class); + $repository = $manager->getRepository(Ticket::class); + + return $repository->findBy( + ['sold' => false], + ['id' => 'ASC'], + $limit, + $offset + ); + } +} diff --git a/tests/Entity/Ticket.php b/tests/Entity/Ticket.php new file mode 100644 index 00000000..e1d17cac --- /dev/null +++ b/tests/Entity/Ticket.php @@ -0,0 +1,84 @@ + false])] + private bool $sold; + + /** + * @param int $id + * @param string $barcode + * @param bool $sold + */ + public function __construct(int $id, string $barcode, bool $sold) + { + $this->id = $id; + $this->barcode = $barcode; + $this->sold = $sold; + } + + public function getId(): int + { + return $this->id; + } + + public function setId(int $id): Ticket + { + $this->id = $id; + + return $this; + } + + public function getBarcode(): string + { + return $this->barcode; + } + + public function setBarcode(string $barcode): Ticket + { + $this->barcode = $barcode; + + return $this; + } + + public function isSold(): bool + { + return $this->sold; + } + + public function setSold(bool $sold): Ticket + { + $this->sold = $sold; + + return $this; + } +} diff --git a/tests/Integration/Command/MeilisearchClearCommandTest.php b/tests/Integration/Command/MeilisearchClearCommandTest.php index 19c4b64c..bac92f2f 100644 --- a/tests/Integration/Command/MeilisearchClearCommandTest.php +++ b/tests/Integration/Command/MeilisearchClearCommandTest.php @@ -36,6 +36,7 @@ public function testClear(): void Cleared sf_phpunit__self_normalizable index of Meilisearch\Bundle\Tests\Entity\SelfNormalizable Cleared sf_phpunit__dummy_custom_groups index of Meilisearch\Bundle\Tests\Entity\DummyCustomGroups Cleared sf_phpunit__dynamic_settings index of Meilisearch\Bundle\Tests\Entity\DynamicSettings +Cleared sf_phpunit__tickets index of Meilisearch\Bundle\Tests\Entity\Ticket Done! EOD, $commandTester->getDisplay()); diff --git a/tests/Integration/Command/MeilisearchCreateCommandTest.php b/tests/Integration/Command/MeilisearchCreateCommandTest.php index 2f2398f9..c0c1bb5c 100644 --- a/tests/Integration/Command/MeilisearchCreateCommandTest.php +++ b/tests/Integration/Command/MeilisearchCreateCommandTest.php @@ -60,6 +60,7 @@ public function testWithoutIndices(bool $updateSettings): void Setting "searchableAttributes" updated of "sf_phpunit__dynamic_settings". Setting "stopWords" updated of "sf_phpunit__dynamic_settings". Setting "synonyms" updated of "sf_phpunit__dynamic_settings". +Creating index sf_phpunit__tickets for Meilisearch\Bundle\Tests\Entity\Ticket Creating index sf_phpunit__aggregated for Meilisearch\Bundle\Tests\Entity\Post Creating index sf_phpunit__aggregated for Meilisearch\Bundle\Tests\Entity\Tag Done! @@ -76,6 +77,7 @@ public function testWithoutIndices(bool $updateSettings): void Creating index sf_phpunit__self_normalizable for Meilisearch\Bundle\Tests\Entity\SelfNormalizable Creating index sf_phpunit__dummy_custom_groups for Meilisearch\Bundle\Tests\Entity\DummyCustomGroups Creating index sf_phpunit__dynamic_settings for Meilisearch\Bundle\Tests\Entity\DynamicSettings +Creating index sf_phpunit__tickets for Meilisearch\Bundle\Tests\Entity\Ticket Creating index sf_phpunit__aggregated for Meilisearch\Bundle\Tests\Entity\Post Creating index sf_phpunit__aggregated for Meilisearch\Bundle\Tests\Entity\Tag Done! diff --git a/tests/Integration/Command/MeilisearchDeleteCommandTest.php b/tests/Integration/Command/MeilisearchDeleteCommandTest.php index 94c00276..6666c6f9 100644 --- a/tests/Integration/Command/MeilisearchDeleteCommandTest.php +++ b/tests/Integration/Command/MeilisearchDeleteCommandTest.php @@ -37,6 +37,7 @@ public function testDeleteWithoutIndices(): void Deleted sf_phpunit__self_normalizable Deleted sf_phpunit__dummy_custom_groups Deleted sf_phpunit__dynamic_settings +Deleted sf_phpunit__tickets Done! EOD, $clearOutput); diff --git a/tests/Integration/Command/MeilisearchImportCommandTest.php b/tests/Integration/Command/MeilisearchImportCommandTest.php index 781a6d5f..63a270b6 100644 --- a/tests/Integration/Command/MeilisearchImportCommandTest.php +++ b/tests/Integration/Command/MeilisearchImportCommandTest.php @@ -391,4 +391,34 @@ public function testAlias(): void self::assertSame(['meili:import'], $command->getAliases()); } + + public function testImportDataProvider(): void + { + $this->createTicket(1, true); + $this->createTicket(2, false); + + $importCommand = $this->application->find('meilisearch:clear'); + $importCommandTester = new CommandTester($importCommand); + $importCommandTester->execute(['--indices' => 'tickets']); + + $importCommand = $this->application->find('meilisearch:import'); + $importCommandTester = new CommandTester($importCommand); + $importCommandTester->execute(['--indices' => 'tickets']); + + $importOutput = $importCommandTester->getDisplay(); + + $this->assertSame(<<<'EOD' +Importing for index Meilisearch\Bundle\Tests\Entity\Ticket +Indexed a batch of 1 / 1 Meilisearch\Bundle\Tests\Entity\Ticket entities into sf_phpunit__tickets index (1 indexed since start) +Done! + +EOD, $importOutput); + + /** @var SearchResult $searchResult */ + $searchResult = $this->client->index($this->getPrefix().'tickets')->search(null); + + $this->assertEquals(1, $searchResult->getHitsCount()); + $this->assertEquals(2, $searchResult->getHit(0)['id']); + $this->assertFalse($searchResult->getHit(0)['sold']); + } } diff --git a/tests/Unit/ConfigurationTest.php b/tests/Unit/ConfigurationTest.php index fca71a73..fc7511cd 100644 --- a/tests/Unit/ConfigurationTest.php +++ b/tests/Unit/ConfigurationTest.php @@ -62,12 +62,18 @@ public function dataTestConfigurationTree(): array [ 'prefix' => 'sf_', 'indices' => [ - ['name' => 'posts', 'class' => 'App\Entity\Post', 'index_if' => null], + [ + 'name' => 'posts', + 'class' => 'App\Entity\Post', + 'index_if' => null, + 'data_provider' => null, + ], [ 'name' => 'tags', 'class' => 'App\Entity\Tag', 'enable_serializer_groups' => true, 'index_if' => null, + 'data_provider' => null, ], ], ], @@ -86,6 +92,7 @@ public function dataTestConfigurationTree(): array 'serializer_groups' => ['searchable'], 'index_if' => null, 'settings' => [], + 'data_provider' => null, ], 1 => [ 'name' => 'tags', @@ -94,6 +101,7 @@ public function dataTestConfigurationTree(): array 'serializer_groups' => ['searchable'], 'index_if' => null, 'settings' => [], + 'data_provider' => null, ], ], ], @@ -108,6 +116,7 @@ public function dataTestConfigurationTree(): array 'enable_serializer_groups' => false, 'index_if' => null, 'settings' => [], + 'data_provider' => null, ], [ 'name' => 'items', @@ -115,6 +124,7 @@ public function dataTestConfigurationTree(): array 'enable_serializer_groups' => false, 'index_if' => null, 'settings' => [], + 'data_provider' => null, ], ], 'nbResults' => 20, @@ -131,7 +141,9 @@ public function dataTestConfigurationTree(): array 'class' => 'App\Entity\Post', 'enable_serializer_groups' => false, 'serializer_groups' => ['searchable'], - 'index_if' => null, 'settings' => [], + 'index_if' => null, + 'settings' => [], + 'data_provider' => null, ], [ 'name' => 'items', @@ -140,6 +152,7 @@ public function dataTestConfigurationTree(): array 'serializer_groups' => ['searchable'], 'index_if' => null, 'settings' => [], + 'data_provider' => null, ], ], 'nbResults' => 20, @@ -159,6 +172,7 @@ public function dataTestConfigurationTree(): array 'serializer_groups' => ['post.public', 'post.private'], 'index_if' => null, 'settings' => [], + 'data_provider' => null, ], ], ], @@ -171,7 +185,9 @@ public function dataTestConfigurationTree(): array 'class' => 'App\Entity\Post', 'enable_serializer_groups' => true, 'serializer_groups' => ['post.public', 'post.private'], - 'index_if' => null, 'settings' => [], + 'index_if' => null, + 'settings' => [], + 'data_provider' => null, ], ], 'nbResults' => 20, @@ -206,6 +222,7 @@ public function dataTestConfigurationTree(): array 'settings' => [ 'distinctAttribute' => ['product_id'], ], + 'data_provider' => null, ], ], 'nbResults' => 20, @@ -240,6 +257,38 @@ public function dataTestConfigurationTree(): array 'settings' => [ 'proximityPrecision' => ['byWord'], ], + 'data_provider' => null, + ], + ], + 'nbResults' => 20, + 'batchSize' => 500, + 'serializer' => 'serializer', + 'doctrineSubscribedEvents' => ['postPersist', 'postUpdate', 'preRemove'], + ], + ], + 'custom data provider' => [ + [ + 'prefix' => 'sf_', + 'indices' => [ + [ + 'name' => 'items', + 'class' => 'App\Entity\Post', + 'data_provider' => 'Meilisearch\Bundle\Tests\DataProvider\TicketDataProvider', + ], + ], + ], + [ + 'url' => 'http://localhost:7700', + 'prefix' => 'sf_', + 'indices' => [ + [ + 'name' => 'items', + 'class' => 'App\Entity\Post', + 'enable_serializer_groups' => false, + 'serializer_groups' => ['searchable'], + 'index_if' => null, + 'settings' => [], + 'data_provider' => 'Meilisearch\Bundle\Tests\DataProvider\TicketDataProvider', ], ], 'nbResults' => 20, diff --git a/tests/Unit/DataProvider/DoctrineOrmDataProviderTest.php b/tests/Unit/DataProvider/DoctrineOrmDataProviderTest.php new file mode 100644 index 00000000..614f3e8e --- /dev/null +++ b/tests/Unit/DataProvider/DoctrineOrmDataProviderTest.php @@ -0,0 +1,24 @@ +get('doctrine')); + + $this->expectException(\Exception::class); + $this->expectExceptionMessage('No entity class name set on data provider.'); + + $dataProvider->getAll(); + } +} diff --git a/tests/config/config.yaml b/tests/config/config.yaml index 87de9ee1..e876a310 100644 --- a/tests/config/config.yaml +++ b/tests/config/config.yaml @@ -24,3 +24,7 @@ doctrine: dir: '%kernel.project_dir%/tests/Entity' prefix: 'Meilisearch\Bundle\Tests\Entity' alias: App + +services: + Meilisearch\Bundle\Tests\DataProvider\TicketDataProvider: + arguments: [ '@doctrine' ] diff --git a/tests/config/config_php7.yaml b/tests/config/config_php7.yaml index 7f690a23..b8d68c4b 100644 --- a/tests/config/config_php7.yaml +++ b/tests/config/config_php7.yaml @@ -28,3 +28,7 @@ doctrine: dir: '%kernel.project_dir%/tests/Entity' prefix: 'Meilisearch\Bundle\Tests\Entity' alias: App + +services: + Meilisearch\Bundle\Tests\DataProvider\TicketDataProvider: + arguments: [ '@doctrine' ] diff --git a/tests/config/meilisearch.yaml b/tests/config/meilisearch.yaml index 50a06b90..25e0c922 100644 --- a/tests/config/meilisearch.yaml +++ b/tests/config/meilisearch.yaml @@ -54,6 +54,9 @@ meilisearch: _service: '@Meilisearch\Bundle\Tests\Integration\Fixtures\StopWords' synonyms: _service: '@Meilisearch\Bundle\Tests\Integration\Fixtures\Synonyms' + - name: tickets + class: 'Meilisearch\Bundle\Tests\Entity\Ticket' + data_provider: 'Meilisearch\Bundle\Tests\DataProvider\TicketDataProvider' services: Meilisearch\Bundle\Tests\Integration\Fixtures\: