Enhanced Entity Serialization and Data Provider Extensibility

src/Command/ImportCommand.php

@@ -4,9 +4,12 @@
 
 namespace ACSEO\TypesenseBundle\Command;
 
+use ACSEO\TypesenseBundle\DataProvider\DataProvider;
 use ACSEO\TypesenseBundle\Manager\CollectionManager;
 use ACSEO\TypesenseBundle\Manager\DocumentManager;
-use ACSEO\TypesenseBundle\Transformer\DoctrineToTypesenseTransformer;
+use ACSEO\TypesenseBundle\Transformer\Transformer;
+use Doctrine\Common\Util\ClassUtils;
+use Doctrine\ORM\AbstractQuery;
 use Doctrine\ORM\EntityManagerInterface;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
@@ -22,6 +25,7 @@ class ImportCommand extends Command
     private $collectionManager;
     private $documentManager;
     private $transformer;
+    private $dataProvider;
     private const ACTIONS = [
         'create',
         'upsert',
@@ -33,13 +37,15 @@ public function __construct(
         EntityManagerInterface $em,
         CollectionManager $collectionManager,
         DocumentManager $documentManager,
-        DoctrineToTypesenseTransformer $transformer
+        Transformer $transformer,
+        DataProvider $dataProvider
     ) {
         parent::__construct();
         $this->em                = $em;
         $this->collectionManager = $collectionManager;
         $this->documentManager   = $documentManager;
         $this->transformer       = $transformer;
+        $this->dataProvider      = $dataProvider;
     }
 
     protected function configure()
@@ -65,8 +71,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             return 1;
         }
 
-        $action = $input->getOption('action');
-
         // 'setMiddlewares' method only exists for Doctrine version >=3.0.0
         if (method_exists($this->em->getConnection()->getConfiguration(), 'setMiddlewares')) {
             $this->em->getConnection()->getConfiguration()->setMiddlewares(
@@ -126,8 +130,8 @@ private function populateIndex(InputInterface $input, OutputInterface $output, s
         $collectionDefinition  = $collectionDefinitions[$index];
         $action                = $input->getOption('action');
 
-        $firstPage  = $input->getOption('first-page');
-        $maxPerPage = $input->getOption('max-per-page');
+        $firstPage  = (int) $input->getOption('first-page');
+        $maxPerPage = (int) $input->getOption('max-per-page');
 
         $collectionName = $collectionDefinition['typesense_name'];
         $class          = $collectionDefinition['entity'];
@@ -151,21 +155,14 @@ private function populateIndex(InputInterface $input, OutputInterface $output, s
 
         $io->text('<info>['.$collectionName.'] '.$class.'</info> '.$nbEntities.' entries to insert splited into '.$nbPages.' pages of '.$maxPerPage.' elements. Insertion from page '.$firstPage.' to '.$lastPage.'.');
 
-        for ($i = $firstPage; $i <= $lastPage; ++$i) {
-            $q = $this->em->createQuery('select e from '.$class.' e')
-                ->setFirstResult(($i - 1) * $maxPerPage)
-                ->setMaxResults($maxPerPage)
-            ;
+        $entityClass = ClassUtils::getRealClass($class);
 
-            if ($io->isDebug()) {
-                $io->text('<info>Running request : </info>'.$q->getSQL());
-            }
-
-            $entities = $q->toIterable();
+        for ($i = $firstPage; $i <= $lastPage; ++$i) {
+            $elements = $this->dataProvider->getData($class, $i, $maxPerPage);
 
             $data = [];
-            foreach ($entities as $entity) {
-                $data[] = $this->transformer->convert($entity);
+            foreach ($elements as $element) {
+                $data[] = $this->transformer->convert($element, $entityClass);
             }
 
             $io->text('Import <info>['.$collectionName.'] '.$class.'</info> Page '.$i.' of '.$lastPage.' ('.count($data).' items)');

src/DataProvider/ArrayDataProvider.php

@@ -0,0 +1,24 @@
+<?php
+
+namespace ACSEO\TypesenseBundle\DataProvider;
+
+use Doctrine\ORM\AbstractQuery;
+use Doctrine\ORM\EntityManagerInterface;
+
+class ArrayDataProvider implements DataProvider
+{
+    private EntityManagerInterface $em;
+
+    public function __construct(EntityManagerInterface $em)
+    {
+        $this->em = $em;
+    }
+
+    public function getData(string $className,  int $page, int $maxPerPage): iterable
+    {
+        return $this->em->createQuery('select e from '.$className.' e')
+            ->setFirstResult(($page - 1) * $maxPerPage)
+            ->setMaxResults($maxPerPage)
+            ->toIterable(hydrationMode: AbstractQuery::HYDRATE_ARRAY);
+    }
+}

src/DataProvider/ContextAwareDataProvider.php

@@ -0,0 +1,8 @@
+<?php
+
+namespace ACSEO\TypesenseBundle\DataProvider;
+
+interface ContextAwareDataProvider extends DataProvider
+{
+    public function supports(string $className);
+}

src/DataProvider/DataProvider.php

@@ -0,0 +1,8 @@
+<?php
+
+namespace ACSEO\TypesenseBundle\DataProvider;
+
+interface DataProvider
+{
+    public function getData(string $className,  int $page, int $maxPerPage): iterable;
+}

src/DataProvider/DataProviderContainer.php

@@ -0,0 +1,30 @@
+<?php
+
+namespace ACSEO\TypesenseBundle\DataProvider;
+
+class DataProviderContainer implements DataProvider
+{
+    /**
+     * @var iterable<ContextAwareDataProvider> $dataProviders
+     */
+    private iterable $dataProviders;
+
+    /**
+     * @param iterable<ContextAwareDataProvider> $dataProviders
+     */
+    public function __construct(iterable $dataProviders)
+    {
+        $this->dataProviders = $dataProviders;
+    }
+
+    public function getData(string $className,  int $page, int $maxPerPage): iterable
+    {
+        foreach ($this->dataProviders as $dataProvider) {
+            if ($dataProvider->supports($className)) {
+                return $dataProvider->getData($className, $page, $maxPerPage);
+            }
+        }
+
+        throw new \RuntimeException('No data provider found for '.$className);
+    }
+}

src/DataProvider/EntityDataProvider.php

@@ -0,0 +1,28 @@
+<?php
+
+namespace ACSEO\TypesenseBundle\DataProvider;
+
+use Doctrine\ORM\EntityManagerInterface;
+
+class EntityDataProvider implements ContextAwareDataProvider
+{
+    private EntityManagerInterface $em;
+
+    public function __construct(EntityManagerInterface $em)
+    {
+        $this->em = $em;
+    }
+
+    public function getData(string $className,  int $page, int $maxPerPage): iterable
+    {
+        return $this->em->createQuery('select e from '.$className.' e')
+            ->setFirstResult(($page - 1) * $maxPerPage)
+            ->setMaxResults($maxPerPage)
+            ->toIterable();
+    }
+
+    public function supports(string $className): bool
+    {
+        return $this->em->getMetadataFactory()->hasMetadataFor($className);
+    }
+}

src/DependencyInjection/ACSEOTypesenseExtension.php

@@ -4,6 +4,8 @@
 
 namespace ACSEO\TypesenseBundle\DependencyInjection;
 
+use ACSEO\TypesenseBundle\DataProvider\ContextAwareDataProvider;
+use ACSEO\TypesenseBundle\Transformer\ContextAwareTransformer;
 use Symfony\Component\Config\FileLocator;
 use Symfony\Component\DependencyInjection\ChildDefinition;
 use Symfony\Component\DependencyInjection\ContainerBuilder;
@@ -61,9 +63,21 @@ public function load(array $configs, ContainerBuilder $container)
         $this->loadFinderServices($container);
 
         $this->loadTransformer($container);
+
+        $this->loadTaggedServices($container);
+
         $this->configureController($container);
     }
 
+    // Add tag to services
+    private function loadTaggedServices(ContainerBuilder $container)
+    {
+        $container->registerForAutoconfiguration(ContextAwareDataProvider::class)
+            ->addTag('typesense.context_aware_data_provider');
+        $container->registerForAutoconfiguration(ContextAwareTransformer::class)
+            ->addTag('typesense.context_aware_transformer');
+    }
+
     /**
      * Loads the configured clients.
      *
@@ -162,6 +176,8 @@ private function loadTransformer(ContainerBuilder $container)
     {
         $managerDef = $container->getDefinition('typesense.transformer.doctrine_to_typesense');
         $managerDef->replaceArgument(0, $this->collectionsConfig);
+        $managerDef = $container->getDefinition('typesense.transformer.array_to_typesense');
+        $managerDef->replaceArgument(0, $this->collectionsConfig);
     }
 
     /**

src/EventListener/TypesenseIndexer.php

@@ -42,8 +42,9 @@ public function postPersist(LifecycleEventArgs $args)
             return;
         }
 
-        $collection = $this->getCollectionName($entity);
-        $data       = $this->transformer->convert($entity);
+        $collection  = $this->getCollectionName($entity);
+        $entityClass = ClassUtils::getClass($entity);
+        $data        = $this->transformer->convert($entity, $entityClass);
 
         $this->documentsToIndex[] = [$collection, $data];
     }
@@ -61,8 +62,10 @@ public function postUpdate(LifecycleEventArgs $args)
 
         $this->checkPrimaryKeyExists($collectionConfig);
 
-        $collection = $this->getCollectionName($entity);
-        $data       = $this->transformer->convert($entity);
+        $collection  = $this->getCollectionName($entity);
+        $entityClass = ClassUtils::getClass($entity);
+
+        $data        = $this->transformer->convert($entity, $entityClass);
 
         $this->documentsToUpdate[] = [$collection, $data['id'], $data];
     }
@@ -85,8 +88,8 @@ public function preRemove(LifecycleEventArgs $args)
         if ($this->entityIsNotManaged($entity)) {
             return;
         }
-
-        $data = $this->transformer->convert($entity);
+        $entityClass = ClassUtils::getClass($entity);
+        $data = $this->transformer->convert($entity, $entityClass);
 
         $this->objetsIdThatCanBeDeletedByObjectHash[spl_object_hash($entity)] = $data['id'];
     }

src/Resources/config/services.xml

@@ -4,7 +4,7 @@
            xsi:schemaLocation="http://symfony.com/schema/dic/services
         https://symfony.com/schema/dic/services/services-1.0.xsd">
     <services>
-        <defaults public="true"/>
+        <defaults public="true" />
 
         <!-- Abstract definition for client. -->
         <service id="typesense.client_prototype" class="ACSEO\TypesenseBundle\Client\TypesenseClient" public="true"
@@ -69,6 +69,26 @@
             <argument type="service" id="service_container" />
         </service>
 
+        <service id="typesense.transformer.array_to_typesense"
+                 class="ACSEO\TypesenseBundle\Transformer\ArrayToTypesenseTransformer">
+            <argument/> <!-- collections -->
+            <tag name="typesense.context_aware_transformer"/>
+        </service>
+
+        <service id="typesense.data_provide.doctrine"
+                 class="ACSEO\TypesenseBundle\DataProvider\EntityDataProvider" autowire="true" >
+        </service>
+
+        <service id="typesense.transformer.container"
+                 class="ACSEO\TypesenseBundle\Transformer\TransformerContainer">
+            <argument type="tagged_iterator" tag="typesense.context_aware_transformer"/>
+        </service>
+
+        <service id="typesense.data_provider.container"
+                 class="ACSEO\TypesenseBundle\DataProvider\DataProviderContainer">
+            <argument type="tagged_iterator" tag="typesense.context_aware_data_provider"/>
+        </service>
+
         <!-- commands -->
         <service id="typesense.command.create" class="ACSEO\TypesenseBundle\Command\CreateCommand" public="true">
             <tag name="console.command"/>
@@ -80,7 +100,8 @@
             <argument type="service" id="doctrine.orm.entity_manager"/>
             <argument type="service" id="typesense.collection_manager"/>
             <argument type="service" id="typesense.document_manager"/>
-            <argument type="service" id="typesense.transformer.doctrine_to_typesense"/>
+            <argument type="service" id="typesense.transformer.container"/>
+            <argument type="service" id="typesense.data_provider.container"/>
         </service>
     </services>
 </container>

src/Transformer/AbstractTransformer.php

@@ -4,7 +4,7 @@
 
 namespace ACSEO\TypesenseBundle\Transformer;
 
-abstract class AbstractTransformer
+abstract class AbstractTransformer implements Transformer
 {
     public const TYPE_COLLECTION   = 'collection';
     public const TYPE_DATETIME     = 'datetime';
@@ -16,23 +16,8 @@ abstract class AbstractTransformer
     public const TYPE_INT_64       = 'int64';
     public const TYPE_BOOL         = 'bool';
 
-    /**
-     * Convert an object to a array of data indexable by typesense.
-     *
-     * @param object $entity the object to convert
-     *
-     * @return array the converted data
-     */
-    abstract public function convert(object $entity): array;
-
-    /**
-     * Convert a value to an acceptable value for typesense.
-     *
-     * @param string $objectClass the object class name
-     * @param string $properyName the property of the object
-     * @param [type] $value the value to convert
-     */
-    abstract public function castValue(string $objectClass, string $properyName, $value);
+    protected array $entityToCollectionMapping;
+    protected array $collectionDefinitions;
 
     /**
      * map a type to a typesense type field.
@@ -57,4 +42,41 @@ public function castType(string $type): string
 
         return $type;
     }
+
+    public function castValue(string $entityClass, string $propertyName, $value)
+    {
+        $collection = $this->entityToCollectionMapping[$entityClass];
+        $key        = array_search(
+            $propertyName,
+            array_column(
+                $this->collectionDefinitions[$collection]['fields'],
+                'name'
+            ), true
+        );
+        $collectionFieldsDefinitions = array_values($this->collectionDefinitions[$collection]['fields']);
+        $originalType                = $collectionFieldsDefinitions[$key]['type'];
+        $castedType                  = $this->castType($originalType);
+
+        switch ($originalType.$castedType) {
+            case self::TYPE_DATETIME.self::TYPE_INT_64:
+                if ($value instanceof \DateTimeInterface) {
+                    return $value->getTimestamp();
+                }
+
+                return null;
+            case self::TYPE_OBJECT.self::TYPE_STRING:
+                return $value->__toString();
+            case self::TYPE_COLLECTION.self::TYPE_ARRAY_STRING:
+                return array_values(
+                    $value->map(function ($v) {
+                        return $v->__toString();
+                    })->toArray()
+                );
+            case self::TYPE_STRING.self::TYPE_STRING:
+            case self::TYPE_PRIMARY.self::TYPE_STRING:
+                return (string) $value;
+            default:
+                return $value;
+        }
+    }
 }