feat: Version 0.20.0 (#228)

* chore: update requirements * feat: site url and clear option for indexer (#227) * feat: use full url if site domain is given * chore: add clear option --------- Co-authored-by: Thessa Kockelkorn <[email protected]> * fix: rename command to fresh * chore: rename clean => fresh * feat: allow document paths (#229) * fix: allow more file paths * feat: show file error in console * style: formatting * feat: tags in overview (#231) * fix: solr accents (#232) * fix: remove double encoding * fix: exclude from search --------- Co-authored-by: thessakockelkorn <[email protected]> Co-authored-by: Thessa Kockelkorn <[email protected]>
NotFoundNL · Apr 18, 2024 · a41666d · a41666d
1 parent e630517
commit a41666d
Show file tree

Hide file tree

Showing 9 changed files with 58 additions and 29 deletions.
diff --git a/composer.json b/composer.json
@@ -20,12 +20,11 @@
         "spatie/laravel-honeypot": "^4.3.2",
         "illuminate/contracts": "^10.0|^11.0",
         "notfoundnl/siteboss-layout": "^1.6.1",
-        "notfoundnl/siteboss-static": "^1.13",
+        "notfoundnl/siteboss-static": "^1.15.0",
         "mcamara/laravel-localization": "^2.0",
         "xenolope/quahog": "^3.0",
         "firebase/php-jwt": "^6.3",
         "intervention/image": "^3.0",
-        "php": "^8.1",
         "doctrine/dbal": "^3.6"
     },
     "require-dev": {

diff --git a/database/migrations/2023_01_12_122527_refactor_attr_menu.php b/database/migrations/2023_01_12_122527_refactor_attr_menu.php
@@ -81,7 +81,7 @@ private function legacyConveryBitToJSon($menuitem)
             'allowChangeParent' => (bool) (intval($menuitem->attr) & $ATTR_CHANGEPARENT),
             'allowDeleteChildren' => ! (intval($menuitem->attr) & $ATTR_NODELETECHILD),
             'allowEnabled' => ! (intval($menuitem->attr) & $ATTR_NOENABLE),
-            'exludeFromSearch' => (bool) $excludeFromSearch,
+            'excludeFromSearch' => (bool) $excludeFromSearch,
         ];
         $menuitem->properties = ($newProperties);
         $menuitem->menu = (bool) (intval($menuitem->attr) & $ATTR_NODELETECHILD);

diff --git a/routes/console.php b/routes/console.php
@@ -5,15 +5,15 @@
 use NotFound\Framework\Services\CmsExchange\ExchangeConsoleService;
 use NotFound\Framework\Services\Indexer\IndexBuilderService;
 
-Artisan::command('siteboss:index-site {--debug : Whether debug messages should be displayed} {--clean : Truncate search table}', function ($debug, $clean) {
+Artisan::command('siteboss:index-site {--debug : Display debug messages} {--fresh : Empty local search table}', function ($debug, $fresh) {
 
-    $indexer = new IndexBuilderService($debug, $clean);
+    $indexer = new IndexBuilderService($debug, $fresh);
     $indexer->run();
 
     return Command::SUCCESS;
 })->purpose('Index site for local search');
 
-Artisan::command('siteboss:cms-import {--debug : Whether debug messages should be displayed} {--dry : Dry Run}', function ($debug, $dry) {
+Artisan::command('siteboss:cms-import {--debug : Display debug messages} {--dry : Dry Run}', function ($debug, $dry) {
 
     $indexer = new ExchangeConsoleService($debug, $dry);
     $indexer->import();

diff --git a/src/Models/Indexes/SolrIndex.php b/src/Models/Indexes/SolrIndex.php
@@ -3,7 +3,6 @@
 namespace NotFound\Framework\Models\Indexes;
 
 use Illuminate\Support\Facades\Mail;
-use Illuminate\Support\Facades\Storage;
 use NotFound\Framework\Mail\Indexer\FileIndexError;
 use NotFound\Framework\Mail\Indexer\QueryError;
 use NotFound\Framework\Models\BaseModel;
@@ -131,14 +130,14 @@ public function testSolrConnection()
         return false;
     }
 
-    public function upsertItem(SearchItem $indexItem, int $siteId = 1): bool
+    public function upsertItem(SearchItem $indexItem, int $siteId = 1, ?string $domain = null): bool
     {
         $curl = $this->solrHandler();
         $doc = [
             sprintf('title_%s', $indexItem->language()) => $indexItem->title(),
             sprintf('content_%s', $indexItem->language()) => html_entity_decode(trim(preg_replace('/\s+/', ' ', preg_replace('#<[^>]+>#', ' ', $indexItem->content())))),
             'type' => $indexItem->type(),
-            'url' => $this->siteUrl($indexItem->url(), $siteId),
+            'url' => $this->siteUrl($indexItem->url(), $domain),
             'priority' => $indexItem->priority(),
             'site' => $siteId,
             'language' => $indexItem->language(),
@@ -197,19 +196,20 @@ public function removeItem($url)
         return false;
     }
 
-    public function upsertFile(SearchItem $indexItem, int $siteId = 1): string
+    public function upsertFile(SearchItem $indexItem, int $siteId = 1, ?string $domain = null): string
     {
 
         // find out of document exists
         $result = 0;
-        $file = Storage::disk('private')->path($indexItem->file());
+        // TODO: check if path is within allowed path
+        $file = $indexItem->file();
 
         if (file_exists($file)) {
             $curl = $this->solrHandler();
             $endpoint = sprintf(
                 '%s/update/extract?literal.url=%s&literal.title_%s=%s&literal.type=%s&literal.site=%s&literal.language=%d&literal.solr_date=%s&uprefix=ignored_&fmap.content=content_%s&commit=true',
                 $this->getSolrBaseUrl(),
-                urlencode($this->siteUrl($indexItem->url(), $siteId)),
+                urlencode($this->siteUrl($indexItem->url(), $domain)),
                 $indexItem->language(),
                 urlencode($indexItem->title()),
                 $indexItem->type(),
@@ -257,6 +257,7 @@ public function upsertFile(SearchItem $indexItem, int $siteId = 1): string
             }
         } else {
             $this->mailFileError($indexItem->title(), $indexItem->url(), 'file bestaat niet');
+            printf("\nFile not found: %s", $file);
 
             return 'fileNotFound';
         }
@@ -441,8 +442,12 @@ public function checkConnection(): bool
         return false;
     }
 
-    public function siteUrl($url, $siteId): string
+    public function siteUrl($url, ?string $domain): string
     {
-        return sprintf('{{%d}}%s', $siteId, $url);
+        if ($domain) {
+            return sprintf('%s/%s', rtrim($domain, '/'), ltrim($url, '/'));
+        }
+
+        return $url;
     }
 }
diff --git a/src/Models/Indexes/SolrItem.php b/src/Models/Indexes/SolrItem.php
@@ -64,7 +64,7 @@ public function __construct($solr, $q, $collate = false, private int $highlightL
         $this->error = isset($solr->error) ? $solr->error : null;
     }
 
-    public function isValid()
+    public function isValid(): bool
     {
         return $this->header && $this->header->status == 0;
     }
@@ -183,21 +183,22 @@ public function predictions()
         return $queries;
     }
 
-    public function spellcheckList()
+    public function spellcheckList(): array
     {
         $items = [];
         $query = $this->q;
 
         if (isset($this->spellcheck)) {
             foreach ($this->spellcheck->suggestions as $suggestion) {
+
                 if (isset($suggestion->startOffset)) {
                     $suggest = substr($query, 0, $suggestion->startOffset).'<em>'.$suggestion->suggestion[0].'</em>'.substr($query, $suggestion->endOffset);
                     $suggestTerm = preg_replace('/^([a-zA-Z])+(_[a-zA-Z]{2})?:/', '', $suggest); // remove search field if necessary
 
                     $suggest_url = substr($query, 0, $suggestion->startOffset).$suggestion->suggestion[0].substr($query, $suggestion->endOffset);
                     $suggest_url = preg_replace('/^([a-zA-Z])+(_[a-zA-Z]{2})?:/', '', $suggest_url); // remove search field if necessary
 
-                    $items[] = (object) ['link' => '?q='.rawurlencode(urldecode($suggest_url)), 'text' => urldecode($suggestTerm)];
+                    $items[] = (object) ['link' => '?q='.rawurlencode(urldecode($suggest_url)), 'text' => $suggestTerm];
                 }
             }
         }

diff --git a/src/Services/Assets/Components/ComponentTags.php b/src/Services/Assets/Components/ComponentTags.php
@@ -5,6 +5,7 @@
 use Illuminate\Support\Facades\DB;
 use NotFound\Framework\Services\Legacy\StatusColumn;
 use NotFound\Layout\Elements\AbstractLayout;
+use NotFound\Layout\Elements\Table\LayoutTableColumn;
 use NotFound\Layout\Inputs\LayoutInputTags;
 
 class ComponentTags extends AbstractComponent
@@ -72,6 +73,19 @@ public function getCurrentValue()
             ->get([$foreignTable.'.'.$p->foreignTagId.' AS id', $foreignTable.'.'.$p->foreignDisplayColumn.' AS label'])->toArray();
     }
 
+    /**
+     * Gets the content for the table overview, this is usually a string.
+     */
+    public function getTableOverviewContent(): LayoutTableColumn
+    {
+        // convert to array with only the value of the label
+        $values = collect($this->getCurrentValue())->map(function ($item) {
+            return $item->label;
+        });
+
+        return new LayoutTableColumn(implode(', ', $values->toArray()));
+    }
+
     /**
      * This function is for doing additional actions while saving, or - when not using the
      * default storage mechanism - doing custom stuff.

diff --git a/src/Services/Indexer/AbstractIndexService.php b/src/Services/Indexer/AbstractIndexService.php
@@ -10,6 +10,8 @@ abstract class AbstractIndexService
 
     public int $siteId;
 
+    public ?string $domain;
+
     abstract public function __construct($debug = false);
 
     abstract public function startUpdate(): bool;
@@ -29,7 +31,7 @@ public function clean(): bool
 
     public function urlNeedsUpdate(string $url, $updated): bool
     {
-        $searchItem = CmsSearch::whereUrl($url)->first();
+        $searchItem = CmsSearch::whereUrl($this->siteUrl($url))->first();
         if ($searchItem && ($searchItem->updated_at !== null && $searchItem->updated_at->timestamp > $updated)) {
             CmsSearch::whereUrl($url)->update(['search_status' => 'SKIPPED']);
 
@@ -41,6 +43,10 @@ public function urlNeedsUpdate(string $url, $updated): bool
 
     protected function siteUrl($url): string
     {
-        return sprintf('{{%d}}%s', $this->siteId, $url);
+        if ($this->domain) {
+            return sprintf('%s/%s', rtrim($this->domain, '/'), ltrim($url, '/'));
+        }
+
+        return $url;
     }
 }
diff --git a/src/Services/Indexer/IndexBuilderService.php b/src/Services/Indexer/IndexBuilderService.php
@@ -12,7 +12,7 @@ class IndexBuilderService
 {
     private bool $debug;
 
-    private bool $clean;
+    private bool $fresh;
 
     private $locales;
 
@@ -22,11 +22,11 @@ class IndexBuilderService
 
     private AbstractIndexService $searchServer;
 
-    public function __construct($debug = false, $clean = false)
+    public function __construct($debug = false, $fresh = false)
     {
         $serverType = config('indexer.engine');
         $this->debug = $debug;
-        $this->clean = $clean;
+        $this->fresh = $fresh;
         $this->locales = Lang::all();
 
         $this->domain = rtrim(env('APP_URL', ''), '/');
@@ -46,7 +46,7 @@ public function run()
 
             return;
         }
-        if ($this->clean) {
+        if ($this->fresh) {
             $this->searchServer->clean();
         }
         $sites = CmsSite::whereIndex(1)->get();
@@ -69,6 +69,8 @@ public function run()
 
                 $siteId = $site->id;
                 $this->searchServer->siteId = $siteId;
+                $this->searchServer->domain = $site->domain ?? null;
+
                 $this->searchServer->languageId = 1;
 
                 // insert all pages, starting from the root
@@ -104,7 +106,7 @@ private function indexChildPages($parentId)
 
                 continue;
             }
-            if (isset($page->properties->exludeFromSearch) && $page->properties->exludeFromSearch == true) {
+            if (isset($page->properties->excludeFromSearch) && $page->properties->excludeFromSearch == true) {
                 $this->writeDebug("  skipping, page not searchable\n");
 
                 continue;

diff --git a/src/Services/Indexer/SolrIndexService.php b/src/Services/Indexer/SolrIndexService.php
@@ -12,6 +12,8 @@ class SolrIndexService extends AbstractIndexService
 
     public int $siteId;
 
+    public ?string $domain;
+
     public $solrIndex;
 
     public function __construct($debug = false)
@@ -26,7 +28,7 @@ public function upsertItem(SearchItem $searchItem): object
         $cmsSearchItemStatus = '';
 
         if ($searchItem->type() === 'file') {
-            $result = $this->solrIndex->upsertFile($searchItem, $this->siteId);
+            $result = $this->solrIndex->upsertFile($searchItem, $this->siteId, $this->domain);
 
             $return = $this->returnvalue();
             if ($result == 'success') {
@@ -37,7 +39,7 @@ public function upsertItem(SearchItem $searchItem): object
                 $return->message = "failed: file not found \n";
             } else {
                 $cmsSearchItemStatus = 'NOT_INDEXABLE';
-                $result = $this->solrIndex->upsertItem($searchItem, $this->siteId);
+                $result = $this->solrIndex->upsertItem($searchItem, $this->siteId, $this->domain);
                 if ($result) {
                     $cmsSearchItemStatus = 'UPDATED';
                 } else {
@@ -46,7 +48,7 @@ public function upsertItem(SearchItem $searchItem): object
                 }
             }
         } else {
-            $result = $this->solrIndex->upsertItem($searchItem, $this->siteId);
+            $result = $this->solrIndex->upsertItem($searchItem, $this->siteId, $this->domain);
 
             if ($result) {
                 $cmsSearchItemStatus = 'UPDATED';
@@ -57,9 +59,9 @@ public function upsertItem(SearchItem $searchItem): object
             }
         }
 
-        $cmsSearchItem = CmsSearch::firstOrNew(['url' => $this->solrIndex->siteUrl($searchItem->url(), $this->siteId)]);
+        $cmsSearchItem = CmsSearch::firstOrNew(['url' => $this->solrIndex->siteUrl($searchItem->url(), $this->domain)]);
         $cmsSearchItem->setValues($searchItem, $cmsSearchItemStatus);
-        $cmsSearchItem->url = $this->solrIndex->siteUrl($searchItem->url(), $this->siteId);
+        $cmsSearchItem->url = $this->solrIndex->siteUrl($searchItem->url(), $this->domain);
         $cmsSearchItem->save();
         if ($cmsSearchItemStatus == 'FAILED') {
             $cmsSearchItem->updated_at = null;