Move partial index into the typo3 database

.github/workflows/ci.yml

@@ -3,12 +3,13 @@ name: CI
 on:
   push:
     branches:
-     - master
+      - master
     tags:
       - '*'
   pull_request:
     branches:
-     - master
+      - master
+      - ES7V2
   schedule:
     - cron: '0 7 * * *'
 

Classes/Controller/BackendController.php

@@ -1,8 +1,7 @@
 <?php
 namespace PAGEmachine\Searchable\Controller;
 
-use PAGEmachine\Searchable\Connection;
-use PAGEmachine\Searchable\Indexer\IndexerFactory;
+use PAGEmachine\Searchable\Domain\Repository\UpdateQueue;
 use PAGEmachine\Searchable\IndexManager;
 use PAGEmachine\Searchable\Query\SearchQuery;
 use PAGEmachine\Searchable\Service\ExtconfService;
@@ -19,17 +18,10 @@
 
 class BackendController extends ActionController
 {
-    /**
-     * @var IndexerFactory $indexerFactory
-     */
-    protected $indexerFactory;
-    public function __construct(private readonly ModuleTemplateFactory $moduleTemplateFactory)
-    {
-    }
-
-    public function injectIndexerFactory(IndexerFactory $indexerFactory): void
-    {
-        $this->indexerFactory = $indexerFactory;
+    public function __construct(
+        private readonly ModuleTemplateFactory $moduleTemplateFactory,
+        private readonly UpdateQueue $updateQueue
+    ) {
     }
 
     /**
@@ -55,25 +47,10 @@ public function startAction(): ResponseInterface
     /**
      * Fetches scheduled updates for backend module
      *
-     * @return array
      */
     protected function fetchScheduledUpdates()
     {
-        $client = Connection::getClient();
-
-        $updates = $client->search([
-            'index' => ExtconfService::getInstance()->getUpdateIndex(),
-            'type' => '',
-            'body' => [
-                'query' => [
-                    'match_all' => new \stdClass(),
-                ],
-            ],
-        ]);
-
-        $updates['count'] = $client->count(['index' => ExtconfService::getInstance()->getUpdateIndex()])['count'];
-
-        return $updates;
+        return $this->updateQueue->pendingUpdates();
     }
 
     /**

Classes/Domain/Repository/UpdateQueue.php

@@ -0,0 +1,70 @@
+<?php
+namespace PAGEmachine\Searchable\Domain\Repository;
+
+/*
+ * This file is part of the PAGEmachine Searchable project.
+ */
+
+use Doctrine\DBAL\Exception\UniqueConstraintViolationException;
+use TYPO3\CMS\Core\Database\ConnectionPool;
+
+final class UpdateQueue
+{
+    private const TABLE_NAME = 'tx_searchable_domain_model_update';
+
+    public function __construct(private readonly ConnectionPool $connectionPool)
+    {
+    }
+
+    public function enqueue(
+        string $type,
+        string $property,
+        int $propertyUid,
+    ): void {
+        try {
+            $this->connectionPool
+                ->getConnectionForTable(self::TABLE_NAME)
+                ->insert(
+                    self::TABLE_NAME,
+                    [
+                        'type' => $type,
+                        'property' => $property,
+                        'property_uid' => $propertyUid,
+                    ],
+                );
+        } catch (UniqueConstraintViolationException) {
+            // Ignore duplicate entry error
+        }
+    }
+
+    public function pendingUpdates(string $type = null): array
+    {
+        $queryBuilder = $this->connectionPool
+            ->getConnectionForTable(self::TABLE_NAME)
+            ->createQueryBuilder();
+
+        $queryBuilder
+            ->select('*')
+            ->from(self::TABLE_NAME);
+
+        if ($type) {
+            $queryBuilder
+                ->where(
+                    $queryBuilder->expr()->eq('type', $queryBuilder->createNamedParameter($type, \PDO::PARAM_STR)),
+                );
+        }
+
+        return $queryBuilder->executeQuery()->fetchAllAssociative();
+    }
+
+    public function clear()
+    {
+        foreach ($this->pendingUpdates() as $object) {
+            $this->connectionPool->getConnectionForTable(self::TABLE_NAME)
+                ->delete(
+                    self::TABLE_NAME,
+                    ['uid' => $object['uid']],
+                );
+        }
+    }
+}

Classes/IndexManager.php

@@ -4,6 +4,7 @@
 use Elasticsearch\Client;
 use PAGEmachine\Searchable\Configuration\ConfigurationManager;
 use PAGEmachine\Searchable\Connection;
+use PAGEmachine\Searchable\Domain\Repository\UpdateQueue;
 use PAGEmachine\Searchable\Service\ConfigurationMergerService;
 use PAGEmachine\Searchable\Service\ExtconfService;
 use TYPO3\CMS\Core\SingletonInterface;
@@ -24,12 +25,15 @@ class IndexManager implements SingletonInterface
      */
     protected $client;
 
+    protected UpdateQueue $updateQueue;
+
     /**
      * @param Client|null $client
      */
-    public function __construct(Client $client = null)
+    public function __construct(Client $client = null, UpdateQueue $updateQueue = null)
     {
         $this->client = $client ?: Connection::getClient();
+        $this->updateQueue = $updateQueue ?: GeneralUtility::makeInstance(UpdateQueue::class);
     }
 
     /**
@@ -116,12 +120,10 @@ public function createIndex($index)
             ],
         ];
 
-        if ($index != ExtconfService::getInstance()->getUpdateIndex()) {
-            $mapping = ConfigurationManager::getInstance()->getMapping($index);
+        $mapping = ConfigurationManager::getInstance()->getMapping($index);
 
-            if (!empty($mapping)) {
-                $params['body']['mappings'] = $mapping;
-            }
+        if (!empty($mapping)) {
+            $params['body']['mappings'] = $mapping;
         }
 
         return $this->client->indices()->create($params);
@@ -134,8 +136,6 @@ public function createIndex($index)
      */
     public function resetUpdateIndex()
     {
-        $this->resetIndex(
-            ExtconfService::getInstance()->getUpdateIndex()
-        );
+        $this->updateQueue->clear();
     }
 }

Classes/Query/UpdateQuery.php

@@ -1,7 +1,8 @@
 <?php
 namespace PAGEmachine\Searchable\Query;
 
-use PAGEmachine\Searchable\Service\ExtconfService;
+use PAGEmachine\Searchable\Domain\Repository\UpdateQueue;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 /*
  * This file is part of the PAGEmachine Searchable project.
@@ -13,13 +14,16 @@
  */
 class UpdateQuery extends AbstractQuery
 {
+    protected UpdateQueue $updateQueue;
+
     /**
      * @return void
      */
     public function __construct()
     {
         parent::__construct();
-        $this->setIndices([ExtconfService::getInstance()->getUpdateIndex()]);
+
+        $this->updateQueue = GeneralUtility::makeInstance(UpdateQueue::class);
 
         $this->init();
     }
@@ -31,35 +35,18 @@ public function __construct()
     public function init()
     {
         $this->parameters =  [
-            'index' => implode(',', $this->getIndices()),
-            'body' => [
-            ],
+            'index' => [],
+            'body' => [],
         ];
     }
 
     /**
      * Adds a new update query string
      *
-     * @return array
      */
-    public function addUpdate($type, $property, $id)
+    public function addUpdate($type, $property, $id): void
     {
-        // Use querystring hash as id to mark each update only once
-        $docid = sha1($type . "." . $property . ":" . $id);
-
-        $this->parameters['id'] = $docid;
-        $this->parameters['type'] = '_doc';
-        $this->parameters['body']['type'] = strval($type);
-        $this->parameters['body']['property'] = $property;
-        $this->parameters['body']['uid'] = $id;
-
-        try {
-            $response = $this->client->index($this->getParameters());
-            return $response;
-        } catch (\Exception $e) {
-            $this->logger->error("Could not track update. Reason: " . $e->getMessage());
-            return [];
-        }
+        $this->updateQueue->enqueue($type, $property, $id);
     }
 
     /**
@@ -69,39 +56,22 @@ public function addUpdate($type, $property, $id)
      */
     public function getUpdates($index, $type)
     {
-        $recordids = [];
-
         $this->init();
 
-        $this->parameters['body'] = [
-            'query' => [
-                'bool' => [
-                    'filter' => [
-                        'term' => [
-                            'type' => $type,
-                        ],
-                    ],
-                ],
-            ],
-        ];
-
-
-        $result = $this->client->search($this->parameters);
+        $recordids = [];
 
-        if (empty($result['hits']['hits'])) {
-            return [];
-        }
+        $results = $this->updateQueue->pendingUpdates($type);
 
         $updateParams = [];
 
-        foreach ($result['hits']['hits'] as $hit) {
+        foreach ($results as $update) {
             //If this is a simple toplevel uid check, we can add this id directly to the updated uid list
-            if ($hit['_source']['property'] == 'uid') {
-                $recordids[$hit['_source']['uid']] = $hit['_source']['uid'];
+            if ($update['property'] == 'uid') {
+                $recordids[$update['property_uid']] = $update['property_uid'];
             } else {
                 $updateParams[] = [
                     "term" => [
-                        $hit['_source']['property'] => $hit['_source']['uid'],
+                        $update['property'] => $update['property_uid'],
                     ],
                 ];
             }
@@ -124,7 +94,7 @@ public function getUpdates($index, $type)
 
             if (!empty($result['hits']['hits'])) {
                 foreach ($result['hits']['hits'] as $hit) {
-                    $recordids[$hit['_id']] = $hit['_id'];
+                    $recordids[$hit['_id']] = (int) $hit['_id'];
                 }
             }
         }

Classes/Service/ExtconfService.php

@@ -225,15 +225,6 @@ public static function getMetaFieldname()
         return $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['searchable']['metaField'];
     }
 
-    /**
-     * Returns the update index
-     * @return string
-     */
-    public function getUpdateIndex()
-    {
-        return $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['searchable']['updateIndex']['name'];
-    }
-
     /**
      * Returns the hosts configuration
      *

Classes/Service/IndexingService.php

@@ -89,11 +89,6 @@ public function setup(): void
         $this->logger->debug('Checking for existing Update Index..');
 
         $indexManager = IndexManager::getInstance();
-        $indexManager->createIndex(
-            ExtconfService::getInstance()->getUpdateIndex()
-        );
-        $this->logger->debug('Ensured update index exists');
-
         $indices = ExtconfService::getIndices();
 
         try {

Configuration/Services.yaml

@@ -37,5 +37,8 @@ services:
   PAGEmachine\Searchable\Service\IndexingService:
     public: true
 
+  PAGEmachine\Searchable\Domain\Repository\UpdateQueue:
+    public: true
+
   PAGEmachine\Searchable\DataCollector\TCA\FormDataRecord:
     autowire: false

Resources/Private/Templates/Backend/Start.html

@@ -43,7 +43,7 @@ <h2>Index information</h2>
 		</div>
 
 		<h2>Update schedule</h2>
-		<p>{updates.count} updates scheduled.</p>
+		<p><f:count>{updates}</f:count> updates scheduled.</p>
 
 		<table class="table">
 			<tr>
@@ -52,11 +52,11 @@ <h2>Update schedule</h2>
 				<th>Uid</th>
 			</tr>
 
-			<f:for each="{updates.hits.hits}" as="update">
+			<f:for each="{updates}" as="update">
 				<tr>
-				 <td>{update._source.type}</td>
-				 <td>{update._source.property}</td>
-				 <td>{update._source.uid}</td>
+				 <td>{update.type}</td>
+				 <td>{update.property}</td>
+				 <td>{update.property_uid}</td>
 				</tr>
 			</f:for>
 

ext_localconf.php

@@ -81,10 +81,6 @@
     // The fieldname to store meta information in (link, preview etc.). This field will be added to all created ES types and set to index = false
     // Note that this field will also affect how you can access the meta fields in templates!
     'metaField' => 'searchable_meta',
-    //Update index. Used for storing the records to update in the next indexing run
-    'updateIndex' => [
-        'name' => 'searchable_updates',
-    ],
     //Add indices here. Default format: language UID => index configuration
     'indices' => [],
     //Add your indexer configurations here. Each indexer represents a toplevel object type like news, pages etc.