perf: optimized rebuilding of items (via chunks)

bimthebam · Sep 13, 2023 · 1bc2e4e · 1bc2e4e
1 parent 7e12851
commit 1bc2e4e
Showing 1 changed file with 24 additions and 10 deletions.
diff --git a/src/Index.php b/src/Index.php
@@ -288,19 +288,33 @@ public function rebuild(): void
             Versioned::set_stage(Versioned::LIVE);
         }
 
-        foreach ($this->getDataList() as $record) {
-            /** @var DataObject|FluentVersionedExtension|FluentExtension $record */
-            if (
-                ($record->hasExtension(FluentVersionedExtension::class) && !$record->isPublishedInLocale())
-                || ($record->hasExtension(FluentExtension::class) && !$record->existsInLocale())
-            ) {
-                continue;
+        $chunkSize = 500;
+        $currentChunk = 0;
+
+        // Keep looping until we run out of chunks
+        while ($chunk = $this->getDataList()->limit($chunkSize, $chunkSize * $currentChunk)->getIterator()) {
+            // Loop over all the item in our chunk
+            foreach ($chunk as $record) {
+                /** @var DataObject|FluentVersionedExtension|FluentExtension $record */
+                if (
+                    ($record->hasExtension(FluentVersionedExtension::class) && !$record->isPublishedInLocale())
+                    || ($record->hasExtension(FluentExtension::class) && !$record->existsInLocale())
+                ) {
+                    continue;
+                }
+
+                $this->add(Document::create($record), false);
             }
 
-            $this->add(Document::create($record), false);
-        }
+            $this->commit();
 
-        $this->commit();
+            if ($chunk->count() < $chunkSize) {
+                // If our last chunk had less item than our chunkSize, we've reach the end.
+                break;
+            }
+
+            $currentChunk++;
+        }
 
         if (
             ClassInfo::exists(Versioned::class)