diff --git a/src/Command/ImportCommand.php b/src/Command/ImportCommand.php index 0aedef8..e9e04a0 100644 --- a/src/Command/ImportCommand.php +++ b/src/Command/ImportCommand.php @@ -4,9 +4,12 @@ namespace ACSEO\TypesenseBundle\Command; +use ACSEO\TypesenseBundle\DataProvider\DataProvider; use ACSEO\TypesenseBundle\Manager\CollectionManager; use ACSEO\TypesenseBundle\Manager\DocumentManager; -use ACSEO\TypesenseBundle\Transformer\DoctrineToTypesenseTransformer; +use ACSEO\TypesenseBundle\Transformer\Transformer; +use Doctrine\Common\Util\ClassUtils; +use Doctrine\ORM\AbstractQuery; use Doctrine\ORM\EntityManagerInterface; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; @@ -22,6 +25,7 @@ class ImportCommand extends Command private $collectionManager; private $documentManager; private $transformer; + private $dataProvider; private const ACTIONS = [ 'create', 'upsert', @@ -33,13 +37,15 @@ public function __construct( EntityManagerInterface $em, CollectionManager $collectionManager, DocumentManager $documentManager, - DoctrineToTypesenseTransformer $transformer + Transformer $transformer, + DataProvider $dataProvider ) { parent::__construct(); $this->em = $em; $this->collectionManager = $collectionManager; $this->documentManager = $documentManager; $this->transformer = $transformer; + $this->dataProvider = $dataProvider; } protected function configure() @@ -65,8 +71,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int return 1; } - $action = $input->getOption('action'); - // 'setMiddlewares' method only exists for Doctrine version >=3.0.0 if (method_exists($this->em->getConnection()->getConfiguration(), 'setMiddlewares')) { $this->em->getConnection()->getConfiguration()->setMiddlewares( @@ -126,8 +130,8 @@ private function populateIndex(InputInterface $input, OutputInterface $output, s $collectionDefinition = $collectionDefinitions[$index]; $action = $input->getOption('action'); - $firstPage = $input->getOption('first-page'); - $maxPerPage = $input->getOption('max-per-page'); + $firstPage = (int) $input->getOption('first-page'); + $maxPerPage = (int) $input->getOption('max-per-page'); $collectionName = $collectionDefinition['typesense_name']; $class = $collectionDefinition['entity']; @@ -151,21 +155,14 @@ private function populateIndex(InputInterface $input, OutputInterface $output, s $io->text('['.$collectionName.'] '.$class.' '.$nbEntities.' entries to insert splited into '.$nbPages.' pages of '.$maxPerPage.' elements. Insertion from page '.$firstPage.' to '.$lastPage.'.'); - for ($i = $firstPage; $i <= $lastPage; ++$i) { - $q = $this->em->createQuery('select e from '.$class.' e') - ->setFirstResult(($i - 1) * $maxPerPage) - ->setMaxResults($maxPerPage) - ; + $entityClass = ClassUtils::getRealClass($class); - if ($io->isDebug()) { - $io->text('Running request : '.$q->getSQL()); - } - - $entities = $q->toIterable(); + for ($i = $firstPage; $i <= $lastPage; ++$i) { + $elements = $this->dataProvider->getData($class, $i, $maxPerPage); $data = []; - foreach ($entities as $entity) { - $data[] = $this->transformer->convert($entity); + foreach ($elements as $element) { + $data[] = $this->transformer->convert($element, $entityClass); } $io->text('Import ['.$collectionName.'] '.$class.' Page '.$i.' of '.$lastPage.' ('.count($data).' items)'); diff --git a/src/DataProvider/ArrayDataProvider.php b/src/DataProvider/ArrayDataProvider.php new file mode 100644 index 0000000..a8f0370 --- /dev/null +++ b/src/DataProvider/ArrayDataProvider.php @@ -0,0 +1,24 @@ +em = $em; + } + + public function getData(string $className, int $page, int $maxPerPage): iterable + { + return $this->em->createQuery('select e from '.$className.' e') + ->setFirstResult(($page - 1) * $maxPerPage) + ->setMaxResults($maxPerPage) + ->toIterable(hydrationMode: AbstractQuery::HYDRATE_ARRAY); + } +} diff --git a/src/DataProvider/ContextAwareDataProvider.php b/src/DataProvider/ContextAwareDataProvider.php new file mode 100644 index 0000000..21fe81a --- /dev/null +++ b/src/DataProvider/ContextAwareDataProvider.php @@ -0,0 +1,8 @@ + $dataProviders + */ + private iterable $dataProviders; + + /** + * @param iterable $dataProviders + */ + public function __construct(iterable $dataProviders) + { + $this->dataProviders = $dataProviders; + } + + public function getData(string $className, int $page, int $maxPerPage): iterable + { + foreach ($this->dataProviders as $dataProvider) { + if ($dataProvider->supports($className)) { + return $dataProvider->getData($className, $page, $maxPerPage); + } + } + + throw new \RuntimeException('No data provider found for '.$className); + } +} diff --git a/src/DataProvider/EntityDataProvider.php b/src/DataProvider/EntityDataProvider.php new file mode 100644 index 0000000..d4244d2 --- /dev/null +++ b/src/DataProvider/EntityDataProvider.php @@ -0,0 +1,28 @@ +em = $em; + } + + public function getData(string $className, int $page, int $maxPerPage): iterable + { + return $this->em->createQuery('select e from '.$className.' e') + ->setFirstResult(($page - 1) * $maxPerPage) + ->setMaxResults($maxPerPage) + ->toIterable(); + } + + public function supports(string $className): bool + { + return $this->em->getMetadataFactory()->hasMetadataFor($className); + } +} diff --git a/src/DependencyInjection/ACSEOTypesenseExtension.php b/src/DependencyInjection/ACSEOTypesenseExtension.php index 5ca9bb0..11ee671 100644 --- a/src/DependencyInjection/ACSEOTypesenseExtension.php +++ b/src/DependencyInjection/ACSEOTypesenseExtension.php @@ -4,6 +4,8 @@ namespace ACSEO\TypesenseBundle\DependencyInjection; +use ACSEO\TypesenseBundle\DataProvider\ContextAwareDataProvider; +use ACSEO\TypesenseBundle\Transformer\ContextAwareTransformer; use Symfony\Component\Config\FileLocator; use Symfony\Component\DependencyInjection\ChildDefinition; use Symfony\Component\DependencyInjection\ContainerBuilder; @@ -61,9 +63,21 @@ public function load(array $configs, ContainerBuilder $container) $this->loadFinderServices($container); $this->loadTransformer($container); + + $this->loadTaggedServices($container); + $this->configureController($container); } + // Add tag to services + private function loadTaggedServices(ContainerBuilder $container) + { + $container->registerForAutoconfiguration(ContextAwareDataProvider::class) + ->addTag('typesense.context_aware_data_provider'); + $container->registerForAutoconfiguration(ContextAwareTransformer::class) + ->addTag('typesense.context_aware_transformer'); + } + /** * Loads the configured clients. * @@ -162,6 +176,8 @@ private function loadTransformer(ContainerBuilder $container) { $managerDef = $container->getDefinition('typesense.transformer.doctrine_to_typesense'); $managerDef->replaceArgument(0, $this->collectionsConfig); + $managerDef = $container->getDefinition('typesense.transformer.array_to_typesense'); + $managerDef->replaceArgument(0, $this->collectionsConfig); } /** diff --git a/src/EventListener/TypesenseIndexer.php b/src/EventListener/TypesenseIndexer.php index b83a032..ed0a9a6 100644 --- a/src/EventListener/TypesenseIndexer.php +++ b/src/EventListener/TypesenseIndexer.php @@ -42,8 +42,9 @@ public function postPersist(LifecycleEventArgs $args) return; } - $collection = $this->getCollectionName($entity); - $data = $this->transformer->convert($entity); + $collection = $this->getCollectionName($entity); + $entityClass = ClassUtils::getClass($entity); + $data = $this->transformer->convert($entity, $entityClass); $this->documentsToIndex[] = [$collection, $data]; } @@ -61,8 +62,10 @@ public function postUpdate(LifecycleEventArgs $args) $this->checkPrimaryKeyExists($collectionConfig); - $collection = $this->getCollectionName($entity); - $data = $this->transformer->convert($entity); + $collection = $this->getCollectionName($entity); + $entityClass = ClassUtils::getClass($entity); + + $data = $this->transformer->convert($entity, $entityClass); $this->documentsToUpdate[] = [$collection, $data['id'], $data]; } @@ -85,8 +88,8 @@ public function preRemove(LifecycleEventArgs $args) if ($this->entityIsNotManaged($entity)) { return; } - - $data = $this->transformer->convert($entity); + $entityClass = ClassUtils::getClass($entity); + $data = $this->transformer->convert($entity, $entityClass); $this->objetsIdThatCanBeDeletedByObjectHash[spl_object_hash($entity)] = $data['id']; } diff --git a/src/Resources/config/services.xml b/src/Resources/config/services.xml index 9b2f11d..4f907a5 100644 --- a/src/Resources/config/services.xml +++ b/src/Resources/config/services.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://symfony.com/schema/dic/services https://symfony.com/schema/dic/services/services-1.0.xsd"> - + + + + + + + + + + + + + + + + + @@ -80,7 +100,8 @@ - + + diff --git a/src/Transformer/AbstractTransformer.php b/src/Transformer/AbstractTransformer.php index 2d3e426..b4ae0ea 100644 --- a/src/Transformer/AbstractTransformer.php +++ b/src/Transformer/AbstractTransformer.php @@ -4,7 +4,7 @@ namespace ACSEO\TypesenseBundle\Transformer; -abstract class AbstractTransformer +abstract class AbstractTransformer implements Transformer { public const TYPE_COLLECTION = 'collection'; public const TYPE_DATETIME = 'datetime'; @@ -16,23 +16,8 @@ abstract class AbstractTransformer public const TYPE_INT_64 = 'int64'; public const TYPE_BOOL = 'bool'; - /** - * Convert an object to a array of data indexable by typesense. - * - * @param object $entity the object to convert - * - * @return array the converted data - */ - abstract public function convert(object $entity): array; - - /** - * Convert a value to an acceptable value for typesense. - * - * @param string $objectClass the object class name - * @param string $properyName the property of the object - * @param [type] $value the value to convert - */ - abstract public function castValue(string $objectClass, string $properyName, $value); + protected array $entityToCollectionMapping; + protected array $collectionDefinitions; /** * map a type to a typesense type field. @@ -57,4 +42,41 @@ public function castType(string $type): string return $type; } + + public function castValue(string $entityClass, string $propertyName, $value) + { + $collection = $this->entityToCollectionMapping[$entityClass]; + $key = array_search( + $propertyName, + array_column( + $this->collectionDefinitions[$collection]['fields'], + 'name' + ), true + ); + $collectionFieldsDefinitions = array_values($this->collectionDefinitions[$collection]['fields']); + $originalType = $collectionFieldsDefinitions[$key]['type']; + $castedType = $this->castType($originalType); + + switch ($originalType.$castedType) { + case self::TYPE_DATETIME.self::TYPE_INT_64: + if ($value instanceof \DateTimeInterface) { + return $value->getTimestamp(); + } + + return null; + case self::TYPE_OBJECT.self::TYPE_STRING: + return $value->__toString(); + case self::TYPE_COLLECTION.self::TYPE_ARRAY_STRING: + return array_values( + $value->map(function ($v) { + return $v->__toString(); + })->toArray() + ); + case self::TYPE_STRING.self::TYPE_STRING: + case self::TYPE_PRIMARY.self::TYPE_STRING: + return (string) $value; + default: + return $value; + } + } } diff --git a/src/Transformer/ArrayToTypesenseTransformer.php b/src/Transformer/ArrayToTypesenseTransformer.php new file mode 100644 index 0000000..d224519 --- /dev/null +++ b/src/Transformer/ArrayToTypesenseTransformer.php @@ -0,0 +1,64 @@ +collectionDefinitions = $collectionDefinitions; + + $this->entityToCollectionMapping = []; + foreach ($this->collectionDefinitions as $collection => $collectionDefinition) { + $this->entityToCollectionMapping[$collectionDefinition['entity']] = $collection; + } + } + + public function convert($element,string $className = null): array + { + foreach ($this->entityToCollectionMapping as $class => $collection) { + if (is_a($className, $class, true)) { + $className = $class; + break; + } + } + + if (!isset($this->entityToCollectionMapping[$className])) { + throw new \Exception(sprintf('Class %s is not supported for Doctrine To Typesense Transformation', $className)); + } + + $data = []; + + $fields = $this->collectionDefinitions[$this->entityToCollectionMapping[$className]]['fields']; + + foreach ($fields as $fieldsInfo) { + try { + $value = $element[$fieldsInfo['entity_attribute']]; + } catch (RuntimeException $exception) { + $value = null; + } + + $name = $fieldsInfo['name']; + + $data[$name] = $this->castValue( + $className, + $name, + $value + ); + } + + return $data; + } + + public function supports(mixed $element, string $className = null) + { + return is_array($element) && $className !== null && isset($this->entityToCollectionMapping[$className]); + } +} diff --git a/src/Transformer/ContextAwareTransformer.php b/src/Transformer/ContextAwareTransformer.php new file mode 100644 index 0000000..6698a1f --- /dev/null +++ b/src/Transformer/ContextAwareTransformer.php @@ -0,0 +1,8 @@ +entityToCollectionMapping) as $class) { - if (is_a($entityClass, $class, true)) { + if (is_a($className, $class, true)) { $entityClass = $class; break; } @@ -71,7 +69,7 @@ public function convert($entity): array $name = $fieldsInfo['name']; $data[$name] = $this->castValue( - $entityClass, + $className, $name, $value ); @@ -141,4 +139,8 @@ private function getFieldValueFromService($entity, $entityAttribute) return null; } + public function supports(mixed $element, string $className = null) + { + return is_object($element) && $className !== null && isset($this->entityToCollectionMapping[$className]); + } } diff --git a/src/Transformer/Transformer.php b/src/Transformer/Transformer.php new file mode 100644 index 0000000..22c88d9 --- /dev/null +++ b/src/Transformer/Transformer.php @@ -0,0 +1,15 @@ + $transformers + */ + private iterable $transformers; + + /** + * @param iterable $transformers + */ + public function __construct(iterable $transformers) + { + $this->transformers = $transformers; + } + + public function convert($element, string $className): array + { + foreach ($this->transformers as $transformer) { + if ($transformer->supports($element, $className)) { + return $transformer->convert($element, $className); + } + } + + throw new \Exception(sprintf('No transformer found for class %s', $className)); + } +} diff --git a/tests/Unit/Transformer/DoctrineToTypesenseTransformerTest.php b/tests/Unit/Transformer/DoctrineToTypesenseTransformerTest.php index 5b4849e..da809c6 100644 --- a/tests/Unit/Transformer/DoctrineToTypesenseTransformerTest.php +++ b/tests/Unit/Transformer/DoctrineToTypesenseTransformerTest.php @@ -275,4 +275,4 @@ private function getContainerInstance() $containerInstance->set('ACSEO\TypesenseBundle\Tests\Functional\Service\ExceptionBookConverter', new ExceptionBookConverter()); return $containerInstance; } -} \ No newline at end of file +}