From f2415f6dca08d5ca29c12e2a1d5ba56249c16e82 Mon Sep 17 00:00:00 2001 From: Xander Schuurman <44030544+keeama13@users.noreply.github.com> Date: Fri, 16 Jun 2023 14:48:29 +0200 Subject: [PATCH 01/20] fix!: field component (#30) --- src/View/Components/Forms/Fields/AbstractFieldComponent.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/View/Components/Forms/Fields/AbstractFieldComponent.php b/src/View/Components/Forms/Fields/AbstractFieldComponent.php index fe68c243..23f9e331 100644 --- a/src/View/Components/Forms/Fields/AbstractFieldComponent.php +++ b/src/View/Components/Forms/Fields/AbstractFieldComponent.php @@ -18,7 +18,7 @@ public function render() return 'siteboss.forms.fields.'.$this->field->type; } - return view('fields::'.$this->field->type); + return view('siteboss::siteboss.forms.fields.'.$this->field->type); } public function colClasses(): string From 2aad4c99a5937f76ae07f4f3cc947f85d7f102ec Mon Sep 17 00:00:00 2001 From: "M.A. Peene" Date: Mon, 19 Jun 2023 14:37:41 +0200 Subject: [PATCH 02/20] Fix: combination render (#31) * fix: render form fields with uppercase in type * fix: use field->GetCombinationChildren to (default) render combination fields * fix: lint --- .../forms/fields/combination.blade.php | 36 ++++++------------- src/Models/Forms/Field.php | 6 ++-- src/Services/Editor/FieldsProperties.php | 2 +- .../Forms/Fields/AbstractFieldComponent.php | 7 ++-- 4 files changed, 18 insertions(+), 33 deletions(-) diff --git a/resources/views/siteboss/forms/fields/combination.blade.php b/resources/views/siteboss/forms/fields/combination.blade.php index c3f2678a..6a827fbe 100644 --- a/resources/views/siteboss/forms/fields/combination.blade.php +++ b/resources/views/siteboss/forms/fields/combination.blade.php @@ -1,26 +1,10 @@ -
-
- {% for item in item.fields %} - {{ include(item.type|lower ~ '.html', {label: item.label, properties: item.properties, id: item.id }) }} - {% endfor %} -
-
- -{% if item.triggerId != '' %} - -{% endif %} +
+ @foreach ($field->GetChildrenOfCombination($field->form_id, $field->id) as $childfield) +
+ +
+ @endforeach +
\ No newline at end of file diff --git a/src/Models/Forms/Field.php b/src/Models/Forms/Field.php index 74a8f38e..48c77bc0 100644 --- a/src/Models/Forms/Field.php +++ b/src/Models/Forms/Field.php @@ -2,6 +2,7 @@ namespace NotFound\Framework\Models\Forms; +use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\SoftDeletes; use NotFound\Framework\Models\BaseModel; @@ -163,7 +164,7 @@ public function getWithChildren(int $formId, int $combinationId = null) return $returnArray; } - public function GetChildrenOfCombination(int $formId, int $combinationId): array + public function GetChildrenOfCombination(int $formId, int $combinationId): Collection { $fieldQuery = $this ->where('form_id', $formId); @@ -174,8 +175,7 @@ public function GetChildrenOfCombination(int $formId, int $combinationId): array return $fieldQuery ->orderBy('order') - ->get() - ->toArray(); + ->get(); } private function setParentRecursive(&$fields, $parentId, $fieldToSet) diff --git a/src/Services/Editor/FieldsProperties.php b/src/Services/Editor/FieldsProperties.php index 3bd1203d..eedf0aea 100644 --- a/src/Services/Editor/FieldsProperties.php +++ b/src/Services/Editor/FieldsProperties.php @@ -150,7 +150,7 @@ private function addLayoutFields(array $properties, LayoutForm &$form) // $checkboxField->setValue(true); // } else { $checkboxField->setValue(false); - // } + // } } else { $checkboxField->setValue($value ?? false); } diff --git a/src/View/Components/Forms/Fields/AbstractFieldComponent.php b/src/View/Components/Forms/Fields/AbstractFieldComponent.php index 23f9e331..8b00ebcc 100644 --- a/src/View/Components/Forms/Fields/AbstractFieldComponent.php +++ b/src/View/Components/Forms/Fields/AbstractFieldComponent.php @@ -14,11 +14,12 @@ public function __construct( public function render() { - if (view()->exists('siteboss.forms.fields.'.$this->field->type)) { - return 'siteboss.forms.fields.'.$this->field->type; + $typeFile = strtolower($this->field->type); + if (view()->exists('siteboss.forms.fields.'.$typeFile)) { + return 'siteboss.forms.fields.'.$typeFile; } - return view('siteboss::siteboss.forms.fields.'.$this->field->type); + return view('siteboss::siteboss.forms.fields.'.$typeFile); } public function colClasses(): string From 5939b00690fb7f42f55e53cdd2b0d3b7ce8a22a6 Mon Sep 17 00:00:00 2001 From: Merijn van Ginkel <107470233+nfmerijn@users.noreply.github.com> Date: Tue, 20 Jun 2023 14:24:28 +0200 Subject: [PATCH 03/20] fix: fixed created_at and updated_at columns not setting values (#32) --- src/Models/Table.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Models/Table.php b/src/Models/Table.php index c0955514..475a300e 100644 --- a/src/Models/Table.php +++ b/src/Models/Table.php @@ -138,6 +138,10 @@ public function updateRecord(array $record): int $tableName = $this->getSiteTableName(); + if (Schema::hasColumn($tableName, 'updated_at')) { + $record['updated_at'] = now(); + } + DB::table($tableName)->where('id', $record['id'])->update($record); return $record['id']; @@ -158,6 +162,10 @@ public function createRecord($record): int $record['status'] = 'PUBLISHED'; } + if (Schema::hasColumn($tableName, 'created_at')) { + $record['created_at'] = now(); + } + DB::table($tableName)->insert($record); return DB::getPdo()->lastInsertId(); From 8df2c337b7fbef15f03f492b1a1a103eed222a13 Mon Sep 17 00:00:00 2001 From: Rene Date: Wed, 21 Jun 2023 15:26:44 +0200 Subject: [PATCH 04/20] feat!: load siteboss routes from normal route file --- routes/api.php | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/routes/api.php b/routes/api.php index 6400ae72..1a993802 100644 --- a/routes/api.php +++ b/routes/api.php @@ -10,7 +10,6 @@ use NotFound\Framework\Http\Controllers\SettingsController; use NotFound\Framework\Http\Controllers\Support\SupportController; use NotFound\Framework\Http\Controllers\UserPreferencesController; -use Siteboss\Routes\SiteRoutes; // ContentBlock /* @@ -39,12 +38,6 @@ // Settings for the login page Route::get('settings', [InfoController::class, 'settings']); - if (class_exists(SiteRoutes::class)) { - Route::group(['middleware' => ['api']], function () { - Route::prefix('public')->group(SiteRoutes::getPublic()); - }); - } - // Authenticated routes Route::group(['middleware' => ['auth:openid', 'api']], function () { // Language for messages (not the language used for storing data) @@ -61,9 +54,10 @@ Route::middleware('role:forms')->group(__DIR__.'/cms/forms.php'); Route::prefix('app')->group(function () { - if (class_exists(SiteRoutes::class)) { - // /site for custom - Route::prefix('site')->group(SiteRoutes::getAppSiteRoutes()); + if (file_exists(base_path().'/routes/siteboss.php')) { + Route::prefix('site')->group( + base_path().'/routes/siteboss.php' + ); } // /menu From 95c68882db26250f9fb96e9fe1e320de89c93a66 Mon Sep 17 00:00:00 2001 From: Thessa Kockelkorn Date: Wed, 21 Jun 2023 15:45:01 +0200 Subject: [PATCH 05/20] feat: language as string, language-specific fields --- src/Models/Indexes/SolrIndex.php | 79 ++++++++++++++----- src/Models/Indexes/SolrItem.php | 13 +-- src/Services/Indexer/AbstractIndexService.php | 4 +- src/Services/Indexer/IndexBuilderService.php | 6 +- src/Services/Indexer/SolrIndexService.php | 5 +- 5 files changed, 72 insertions(+), 35 deletions(-) diff --git a/src/Models/Indexes/SolrIndex.php b/src/Models/Indexes/SolrIndex.php index a795d8d4..db150443 100644 --- a/src/Models/Indexes/SolrIndex.php +++ b/src/Models/Indexes/SolrIndex.php @@ -83,7 +83,7 @@ public function emptyCore() $json = json_decode($result); - if (! $json || ! isset($json->responseHeader) || $json->responseHeader->status !== 0) { + if (!$json || !isset($json->responseHeader) || $json->responseHeader->status !== 0) { $this->mailQueryError($url, $result); return false; @@ -100,7 +100,7 @@ private function solrHandler() { $handler = curl_init(); curl_setopt($handler, CURLOPT_RETURNTRANSFER, true); - curl_setopt($handler, CURLOPT_USERPWD, $this->solrUser.':'.$this->solrPass); + curl_setopt($handler, CURLOPT_USERPWD, $this->solrUser . ':' . $this->solrPass); curl_setopt($handler, CURLOPT_POST, true); @@ -130,13 +130,13 @@ public function testSolrConnection() return false; } - public function addOrUpdateItem(string $url, string $title, string $contents, string $type, int $lang, int $siteId, array $customValues, int $priority): bool + public function addOrUpdateItem(string $url, string $title, string $contents, string $type, string $lang, int $siteId, array $customValues, int $priority): bool { $curl = $this->solrHandler(); $doc = [ - 'title' => $title, - 'content' => html_entity_decode(trim(preg_replace('/\s+/', ' ', strip_tags($contents)))), + sprintf('title_%s', $lang) => $title, + sprintf('content_%s', $lang) => html_entity_decode(trim(preg_replace('/\s+/', ' ', strip_tags($contents)))), 'type' => $type, 'url' => $url, 'priority' => $priority, @@ -167,7 +167,7 @@ public function addOrUpdateItem(string $url, string $title, string $contents, st public function removeItem($url) { - if (! is_null($url)) { + if (!is_null($url)) { $curl = $this->solrHandler(); $payload = ['delete' => $url]; @@ -192,7 +192,7 @@ public function removeItem($url) return false; } - public function addOrUpdateFile(string $url, string $title, string $file, string $type, int $lang, int $siteId, array $customValues, int $priority): string + public function addOrUpdateFile(string $url, string $title, string $file, string $type, string $lang, int $siteId, array $customValues, int $priority): string { // find out of document exists $result = 0; @@ -202,9 +202,10 @@ public function addOrUpdateFile(string $url, string $title, string $file, string $curl = $this->solrHandler(); $endpoint = sprintf( - '%s/update/extract?literal.url=%s&literal.title=%s&literal.type=%s&literal.site=%s&literal.language=%d&commit=true', + '%s/update/extract?literal.url=%s&literal.title_%s=%s&literal.type=%s&literal.site=%s&literal.language=%d&commit=true', $this->getSolrBaseUrl(), urlencode($url), + $lang, urlencode($title), $type, $siteId, @@ -268,46 +269,48 @@ private function mailQueryError($query, $result) } } - public function selectItems($query, $filter = null, $start = null, $rows = null, $extraColumns = [], $highlightLength = 50) + public function selectItems($query, $lang = 'nl', $filter = null, $start = null, $rows = null, $extraColumns = [], $highlightLength = 50) { $curl = $this->solrHandler(); $url = sprintf( - '%s/select?q=title:%s%%20content:%s&spellcheck.q=%s&wt=%s&hl=%s&q.op=%s&hl.fl=%s&fl=%s&spellcheck=true&hl.fragsize=%d&hl.maxAnalyzedChars=%d', + '%s/select?q=title_%s:%s%%20content_%s:%s&spellcheck.q=%s&wt=%s&hl=%s&q.op=%s&hl.fl=%s&fl=%s&spellcheck=true&hl.fragsize=%d&hl.maxAnalyzedChars=%d&spellcheck.dictionary=spellcheck_%s', $this->getSolrBaseUrl(), + sprintf($lang), + rawurlencode($query), // make sure + between search terms is preserved + sprintf($lang), rawurlencode($query), // make sure + between search terms is preserved rawurlencode($query), // make sure + between search terms is preserved - rawurlencode($query), // make sure + between search terms is preserved rawurlencode($query), // make sure + between search terms is preserved $this->wt, $this->hl, $this->selectOperator, - $this->hlfl, + sprintf('%s_%s', $this->hlfl, $lang), urlencode($this->fl), $this->hlfragsize, $this->hlmaxAnalyzedChars, + $lang ); if ($filter) { - $url .= '&fq='.$filter; + $url .= '&fq=' . $filter; } if ($start && is_int($start)) { - $url .= '&start='.$start; + $url .= '&start=' . $start; } if ($rows && is_int($rows)) { - $url .= '&rows='.$rows; + $url .= '&rows=' . $rows; } if (count($extraColumns) > 0) { } if ($this->sort) { - $url .= '&sort='.urlencode($this->sort); + $url .= '&sort=' . urlencode($this->sort); } - curl_setopt($curl, CURLOPT_URL, $url); $result = curl_exec($curl); $json = json_decode($result); $searchResults = new SolrItem($json, $query, false, $highlightLength); - if (! $searchResults->isValid()) { + if (!$searchResults->isValid()) { $this->mailQueryError($url, $result); } @@ -327,7 +330,7 @@ public function suggestItems($query, $filter = null) $result = curl_exec($curl); $json = json_decode($result); $suggestions = new SolrItem($json, $query); - if (! $suggestions->isValid()) { + if (!$suggestions->isValid()) { $this->buildSuggester(); $result = curl_exec($curl); $json = json_decode($result); @@ -359,7 +362,6 @@ public function buildSuggester() $url = sprintf('%s&suggest.build=true', $this->suggestUrl()); curl_setopt($curl, CURLOPT_URL, $url); - $result = curl_exec($curl); $json = json_decode($result); $searchResults = new SolrItem($json, null); @@ -367,10 +369,45 @@ public function buildSuggester() return $searchResults; } + private function getConfig() + { + $curl = $this->solrHandler(); + $url = sprintf('%s/config/searchComponent?componentName=suggest', $this->getSolrBaseUrl()); + curl_setopt($curl, CURLOPT_URL, $url); + curl_setopt($curl, CURLOPT_POST, false); + + $result = curl_exec($curl); + $json = json_decode($result); + + return $json; + } + + private function allSuggesters() + { + $json = $this->getConfig(); + $suggesters = []; + if ( + $json && isset($json->responseHeader) + && $json->responseHeader->status == 0 + && isset($json->config->searchComponent->suggest->suggester) + ) { + $list = $json->config->searchComponent->suggest->suggester; + foreach ($list as $s) { + $suggesters[] = $s->name; + } + } + + return $suggesters; + } + private function explodeSuggesters(): string { $suggesterString = ''; - foreach ($this->suggester as $s) { + $suggesters = $this->suggester; + if (count($suggesters) == 0) { + $suggesters = $this->allSuggesters(); + } + foreach ($suggesters as $s) { $suggesterString .= sprintf('&suggest.dictionary=%s', $s); } diff --git a/src/Models/Indexes/SolrItem.php b/src/Models/Indexes/SolrItem.php index ad627228..7df6ae1f 100644 --- a/src/Models/Indexes/SolrItem.php +++ b/src/Models/Indexes/SolrItem.php @@ -41,7 +41,7 @@ public function __construct($solr, $q, $collate = false, private int $highlightL $this->fl = explode(' ', $fl); $this->results = isset($solr->response->docs) ? $solr->response->docs : null; $this->highlights = isset($solr->highlighting) ? $solr->highlighting : null; - $this->spellcheck = $solr->spellcheck ?? null; + $this->spellcheck = isset($solr->spellcheck) ? $solr->spellcheck : null; $this->suggest = isset($solr->suggest) ? $solr->suggest : null; $this->number = isset($solr->response->numFound) ? $solr->response->numFound : 0; @@ -84,7 +84,8 @@ public function resultList() if ($column == 'url') { $resultArray[$column] = $this->parseUrl($result->{$column}); } else { - $resultArray[$column] = isset($result->{$column}[0]) ? $result->{$column}[0] : ''; + $columnName = preg_replace('/(_[a-zA-Z]{2}$)/', '', $column); + $resultArray[$columnName] = isset($result->{$column}) ? $result->{$column} : ''; } } @@ -180,18 +181,18 @@ public function predictions() public function spellcheckList() { $items = []; - $query = $this->q; + $query = $this->header->params->q; if (isset($this->spellcheck)) { foreach ($this->spellcheck->suggestions as $suggestion) { if (isset($suggestion->startOffset)) { $suggest = substr($query, 0, $suggestion->startOffset).''.$suggestion->suggestion[0].''.substr($query, $suggestion->endOffset); - $suggest = preg_replace('/^([a-zA-Z])+:/', '', $suggest); // remove search field if necessary + $suggestTerm = preg_replace('/^([a-zA-Z])+(_[a-zA-Z]{2})?:/', '', $suggest); // remove search field if necessary $suggest_url = substr($query, 0, $suggestion->startOffset).$suggestion->suggestion[0].substr($query, $suggestion->endOffset); - $suggest_url = preg_replace('/^([a-zA-Z])+:/', '', $suggest_url); // remove search field if necessary + $suggest_url = preg_replace('/^([a-zA-Z])+(_[a-zA-Z]{2})?:/', '', $suggest_url); // remove search field if necessary - $items[] = (object) ['link' => '?q='.rawurlencode(urldecode($suggest_url)), 'text' => urldecode($suggest)]; + $items[] = (object) ['link' => '?q='.rawurlencode(urldecode($suggest_url)), 'text' => urldecode($suggestTerm)]; } } } diff --git a/src/Services/Indexer/AbstractIndexService.php b/src/Services/Indexer/AbstractIndexService.php index c142af30..0add32a8 100644 --- a/src/Services/Indexer/AbstractIndexService.php +++ b/src/Services/Indexer/AbstractIndexService.php @@ -18,7 +18,7 @@ abstract public function finishUpdate(): object; abstract public function urlNeedsUpdate(string $url, $updated): bool; - abstract public function upsertUrl(string $url, string $title, string $contents, string $type, int $lang, array $customValues = []): object; + abstract public function upsertUrl(string $url, string $title, string $contents, string $type, string $lang, array $customValues = []): object; - abstract public function upsertFile(string $url, string $title, string $file, string $type, int $lang, array $customValues): object; + abstract public function upsertFile(string $url, string $title, string $file, string $type, string $lang, array $customValues): object; } diff --git a/src/Services/Indexer/IndexBuilderService.php b/src/Services/Indexer/IndexBuilderService.php index c77fa385..a1c5aafb 100644 --- a/src/Services/Indexer/IndexBuilderService.php +++ b/src/Services/Indexer/IndexBuilderService.php @@ -159,7 +159,7 @@ private function updatePage($menu, $lang) $searchText = rtrim($searchText, ', '); if (! empty($title) && ! empty($searchText)) { - $result = $this->searchServer->upsertUrl($url, $title, $searchText, 'page', $lang->id, $customValues, $priority); + $result = $this->searchServer->upsertUrl($url, $title, $searchText, 'page', $lang->url, $customValues, $priority); if ($result->errorCode == 0) { $this->writeDebug(" success\n"); @@ -207,9 +207,9 @@ private function updateSubitems($class, $lang) $success = true; if ($searchItem['isFile']) { - $success = $this->searchServer->upsertFile($url, $searchItem['title'], $searchItem['file'], $searchItem['type'], $lang->id, $searchItem['customValues'], $searchItem['priority']); + $success = $this->searchServer->upsertFile($url, $searchItem['title'], $searchItem['file'], $searchItem['type'], $lang->url, $searchItem['customValues'], $searchItem['priority']); } else { // subitem is table row - $success = $this->searchServer->upsertUrl($url, $searchItem['title'], $searchItem['content'], $searchItem['type'], $lang->id, $searchItem['customValues'], $searchItem['priority']); + $success = $this->searchServer->upsertUrl($url, $searchItem['title'], $searchItem['content'], $searchItem['type'], $lang->url, $searchItem['customValues'], $searchItem['priority']); } if ($this->sitemapFile && $searchItem['sitemap']) { diff --git a/src/Services/Indexer/SolrIndexService.php b/src/Services/Indexer/SolrIndexService.php index 92aa1dc9..22c59b8b 100644 --- a/src/Services/Indexer/SolrIndexService.php +++ b/src/Services/Indexer/SolrIndexService.php @@ -32,7 +32,7 @@ public function urlNeedsUpdate(string $url, $updated): bool return true; } - public function upsertUrl(string $url, string $title, string $contents, string $type, int $lang, array $customValues = [], $priority = 1): object + public function upsertUrl(string $url, string $title, string $contents, string $type, string $lang, array $customValues = [], $priority = 1): object { $result = $this->solrIndex->addOrUpdateItem($this->siteUrl($url), $title, $contents, $type, $lang, $this->siteId, $customValues, $priority); $return = $this->returnvalue(); @@ -51,7 +51,7 @@ public function upsertUrl(string $url, string $title, string $contents, string $ return $return; } - public function upsertFile(string $url, string $title, string $file, string $type, int $lang, array $customValues = [], $priority = 1): object + public function upsertFile(string $url, string $title, string $file, string $type, string $lang, array $customValues = [], $priority = 1): object { $result = $this->solrIndex->addOrUpdateFile($this->siteUrl($url), $title, $file, $type, $lang, $this->siteId, $customValues, $priority); @@ -107,7 +107,6 @@ public function finishUpdate(): object { $return = $this->removeAllPending(); - $this->solrIndex->suggester = ['fulltextsuggester', 'titlesuggester']; $build = $this->solrIndex->buildSuggester(); if ($build->error) { From bda113d004bae6aed7eeb860976f2ab8f035b80b Mon Sep 17 00:00:00 2001 From: Thessa Kockelkorn Date: Wed, 21 Jun 2023 15:48:24 +0200 Subject: [PATCH 06/20] fix: right query for spellcheck --- src/Models/Indexes/SolrItem.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Models/Indexes/SolrItem.php b/src/Models/Indexes/SolrItem.php index 7df6ae1f..f6b91cec 100644 --- a/src/Models/Indexes/SolrItem.php +++ b/src/Models/Indexes/SolrItem.php @@ -110,7 +110,7 @@ private function parseUrl($rawurl) public function getSuggestions() { $suggestions = $this->suggestions(); - if (! $suggestions) { + if (!$suggestions) { $suggestions = $this->collations(); } @@ -149,7 +149,7 @@ public function collations() } $queries[] = [ 'term' => implode(' ', $emphasizedArray), - 'payload' => '?q='.urlencode($querytext).'', + 'payload' => '?q=' . urlencode($querytext) . '', ]; } } @@ -167,10 +167,10 @@ public function predictions() $next = next($this->predictions); if (is_string($current) && is_int($next)) { $resultList[$current]['count'] = $next; - $fullTerm = $term.' '.$current; + $fullTerm = $term . ' ' . $current; $queries[] = [ 'term' => $fullTerm, - 'payload' => '?q='.urlencode($fullTerm).'', + 'payload' => '?q=' . urlencode($fullTerm) . '', ]; } } @@ -181,18 +181,18 @@ public function predictions() public function spellcheckList() { $items = []; - $query = $this->header->params->q; + $query = $this->q; if (isset($this->spellcheck)) { foreach ($this->spellcheck->suggestions as $suggestion) { if (isset($suggestion->startOffset)) { - $suggest = substr($query, 0, $suggestion->startOffset).''.$suggestion->suggestion[0].''.substr($query, $suggestion->endOffset); + $suggest = substr($query, 0, $suggestion->startOffset) . '' . $suggestion->suggestion[0] . '' . substr($query, $suggestion->endOffset); $suggestTerm = preg_replace('/^([a-zA-Z])+(_[a-zA-Z]{2})?:/', '', $suggest); // remove search field if necessary - $suggest_url = substr($query, 0, $suggestion->startOffset).$suggestion->suggestion[0].substr($query, $suggestion->endOffset); + $suggest_url = substr($query, 0, $suggestion->startOffset) . $suggestion->suggestion[0] . substr($query, $suggestion->endOffset); $suggest_url = preg_replace('/^([a-zA-Z])+(_[a-zA-Z]{2})?:/', '', $suggest_url); // remove search field if necessary - $items[] = (object) ['link' => '?q='.rawurlencode(urldecode($suggest_url)), 'text' => urldecode($suggestTerm)]; + $items[] = (object) ['link' => '?q=' . rawurlencode(urldecode($suggest_url)), 'text' => urldecode($suggestTerm)]; } } } From 1f39c7ed425bb0d9cc9dde2f81df2bf1f924b847 Mon Sep 17 00:00:00 2001 From: Thessa Kockelkorn Date: Mon, 26 Jun 2023 14:38:51 +0200 Subject: [PATCH 07/20] fix: variable names --- resources/views/mail/indexer/file-index-error.blade.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/views/mail/indexer/file-index-error.blade.php b/resources/views/mail/indexer/file-index-error.blade.php index 57beb5ca..f2996563 100644 --- a/resources/views/mail/indexer/file-index-error.blade.php +++ b/resources/views/mail/indexer/file-index-error.blade.php @@ -1,5 +1,5 @@ @component('mail::message') -

Document {{ document }} op server {{ server }} geeft de volgende fout: {{ error }}

+

Document {{ $document }} op server {{ $server }} geeft de volgende fout: {{ $error }}

@endcomponent \ No newline at end of file From 33deaef69a46d47168a21a179daf37bfad51902c Mon Sep 17 00:00:00 2001 From: Thessa Kockelkorn Date: Mon, 26 Jun 2023 15:35:08 +0200 Subject: [PATCH 08/20] fix: passing of language url --- src/Models/Indexes/SolrIndex.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Models/Indexes/SolrIndex.php b/src/Models/Indexes/SolrIndex.php index db150443..547991bf 100644 --- a/src/Models/Indexes/SolrIndex.php +++ b/src/Models/Indexes/SolrIndex.php @@ -275,9 +275,9 @@ public function selectItems($query, $lang = 'nl', $filter = null, $start = null, $url = sprintf( '%s/select?q=title_%s:%s%%20content_%s:%s&spellcheck.q=%s&wt=%s&hl=%s&q.op=%s&hl.fl=%s&fl=%s&spellcheck=true&hl.fragsize=%d&hl.maxAnalyzedChars=%d&spellcheck.dictionary=spellcheck_%s', $this->getSolrBaseUrl(), - sprintf($lang), + $lang, rawurlencode($query), // make sure + between search terms is preserved - sprintf($lang), + $lang, rawurlencode($query), // make sure + between search terms is preserved rawurlencode($query), // make sure + between search terms is preserved $this->wt, From 072f6a252d5a7e69206c9f24ae90a52879f28a50 Mon Sep 17 00:00:00 2001 From: Thessa Kockelkorn Date: Tue, 27 Jun 2023 12:41:43 +0200 Subject: [PATCH 09/20] fix: pass page id for custom search values --- src/Services/Indexer/IndexBuilderService.php | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Services/Indexer/IndexBuilderService.php b/src/Services/Indexer/IndexBuilderService.php index a1c5aafb..ccba051a 100644 --- a/src/Services/Indexer/IndexBuilderService.php +++ b/src/Services/Indexer/IndexBuilderService.php @@ -44,7 +44,7 @@ public function run() if (count($sites) > 0) { $startResult = $this->searchServer->startUpdate(); - if (! $startResult) { + if (!$startResult) { $this->writeDebug("\n\n Error when emptying core! \n\n"); } @@ -85,13 +85,13 @@ private function indexChildPages($parentId) foreach ($childPages as $page) { $this->writeDebug(sprintf(" * Page \e[1m%s\e[0m (id: %d)", $page->url, $page->id)); - if (! isset($page->template->id)) { + if (!isset($page->template->id)) { $this->writeDebug(" skipping, no template found\n"); continue; } - if (! isset($page->template->properties->searchable) || $page->template->properties->searchable == 0) { + if (!isset($page->template->properties->searchable) || $page->template->properties->searchable == 0) { $this->writeDebug(" skipping, template not searchable\n"); continue; @@ -144,13 +144,13 @@ private function updatePage($menu, $lang) $customValues = []; $class = $menu->template->filename ?? ''; - $className = 'App\Http\Controllers\Page\\'.$class.'Controller'; + $className = 'App\Http\Controllers\Page\\' . $class . 'Controller'; $c = null; $priority = 1; if (class_exists($className)) { $c = new $className(); if (method_exists($className, 'customSearchValues')) { - $customValues = $c->customSearchValues(); + $customValues = $c->customSearchValues($menu->id); } if (method_exists($className, 'searchPriority')) { $priority = $c->searchPriority(); @@ -158,7 +158,7 @@ private function updatePage($menu, $lang) } $searchText = rtrim($searchText, ', '); - if (! empty($title) && ! empty($searchText)) { + if (!empty($title) && !empty($searchText)) { $result = $this->searchServer->upsertUrl($url, $title, $searchText, 'page', $lang->url, $customValues, $priority); if ($result->errorCode == 0) { @@ -183,7 +183,7 @@ private function updatePage($menu, $lang) private function updateSubPages($menu, $lang) { $class = $menu->template->filename ?? ''; - $className = 'App\Http\Controllers\Page\\'.$class.'Controller'; + $className = 'App\Http\Controllers\Page\\' . $class . 'Controller'; $c = null; // update subPage if necessary @@ -235,8 +235,8 @@ private function updateSubitems($class, $lang) private function createFolderIfNotExists($fullFilePath) { $path_parts = pathinfo($fullFilePath); - if (! file_exists($path_parts['dirname'])) { - if (! mkdir($path_parts['dirname'])) { + if (!file_exists($path_parts['dirname'])) { + if (!mkdir($path_parts['dirname'])) { printf("\n\n### Error creating sitemap folder"); } } From 2509a80db0770a8c178393723362fc50102e6bcd Mon Sep 17 00:00:00 2001 From: Thessa Kockelkorn Date: Wed, 28 Jun 2023 14:43:03 +0200 Subject: [PATCH 10/20] feat: optional sorting of search results --- src/Models/Indexes/SolrIndex.php | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Models/Indexes/SolrIndex.php b/src/Models/Indexes/SolrIndex.php index 547991bf..4c0ea52d 100644 --- a/src/Models/Indexes/SolrIndex.php +++ b/src/Models/Indexes/SolrIndex.php @@ -133,7 +133,6 @@ public function testSolrConnection() public function addOrUpdateItem(string $url, string $title, string $contents, string $type, string $lang, int $siteId, array $customValues, int $priority): bool { $curl = $this->solrHandler(); - $doc = [ sprintf('title_%s', $lang) => $title, sprintf('content_%s', $lang) => html_entity_decode(trim(preg_replace('/\s+/', ' ', strip_tags($contents)))), @@ -157,6 +156,7 @@ public function addOrUpdateItem(string $url, string $title, string $contents, st curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($payload)); $result = curl_exec($curl); + $json = json_decode($result); if ($json && isset($json->responseHeader) && $json->responseHeader->status == 0) { return true; @@ -269,7 +269,7 @@ private function mailQueryError($query, $result) } } - public function selectItems($query, $lang = 'nl', $filter = null, $start = null, $rows = null, $extraColumns = [], $highlightLength = 50) + public function selectItems($query, $lang = 'nl', $filter = null, $start = null, $rows = null, $extraColumns = [], $highlightLength = 50, $sortField = NULL, $sortDirection = 'desc') { $curl = $this->solrHandler(); $url = sprintf( @@ -299,13 +299,15 @@ public function selectItems($query, $lang = 'nl', $filter = null, $start = null, if ($rows && is_int($rows)) { $url .= '&rows=' . $rows; } - if (count($extraColumns) > 0) { } - if ($this->sort) { - $url .= '&sort=' . urlencode($this->sort); + if ($sortField) { + $url .= '&sort=' . urlencode($sortField . " " . $sortDirection); } + + echo $url; + curl_setopt($curl, CURLOPT_URL, $url); $result = curl_exec($curl); $json = json_decode($result); From 29682652c4e9059363c43b0dcd4c37f9c0e51776 Mon Sep 17 00:00:00 2001 From: Thessa Kockelkorn Date: Fri, 30 Jun 2023 13:34:53 +0200 Subject: [PATCH 11/20] fix: remove debug --- src/Models/Indexes/SolrIndex.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Models/Indexes/SolrIndex.php b/src/Models/Indexes/SolrIndex.php index 4c0ea52d..d868fd7b 100644 --- a/src/Models/Indexes/SolrIndex.php +++ b/src/Models/Indexes/SolrIndex.php @@ -306,8 +306,6 @@ public function selectItems($query, $lang = 'nl', $filter = null, $start = null, $url .= '&sort=' . urlencode($sortField . " " . $sortDirection); } - echo $url; - curl_setopt($curl, CURLOPT_URL, $url); $result = curl_exec($curl); $json = json_decode($result); From 288ee097a5211761f0bd0135170aacb42afea7f4 Mon Sep 17 00:00:00 2001 From: Thessa Kockelkorn Date: Fri, 30 Jun 2023 13:52:35 +0200 Subject: [PATCH 12/20] fix: change language column into string --- ...49_change_search_table_language_column.php | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 database/migrations/2023_06_30_134549_change_search_table_language_column.php diff --git a/database/migrations/2023_06_30_134549_change_search_table_language_column.php b/database/migrations/2023_06_30_134549_change_search_table_language_column.php new file mode 100644 index 00000000..dee3acb2 --- /dev/null +++ b/database/migrations/2023_06_30_134549_change_search_table_language_column.php @@ -0,0 +1,27 @@ +string('language', 64)->change(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + // + } +}; From 2550ac47b02cc532fb14d4530e525f9881b974f6 Mon Sep 17 00:00:00 2001 From: Thessa Kockelkorn Date: Mon, 3 Jul 2023 11:28:19 +0200 Subject: [PATCH 13/20] feat: use config for sitemap --- src/Services/Indexer/IndexBuilderService.php | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Services/Indexer/IndexBuilderService.php b/src/Services/Indexer/IndexBuilderService.php index ccba051a..5d769117 100644 --- a/src/Services/Indexer/IndexBuilderService.php +++ b/src/Services/Indexer/IndexBuilderService.php @@ -44,13 +44,13 @@ public function run() if (count($sites) > 0) { $startResult = $this->searchServer->startUpdate(); - if (!$startResult) { + if (! $startResult) { $this->writeDebug("\n\n Error when emptying core! \n\n"); } foreach ($sites as $site) { $siteName = $site->name; - $sitemapFileName = env('APP_SITEMAP'); + $sitemapFileName = config('solr.sitemap'); if ($sitemapFileName) { $this->createFolderIfNotExists($sitemapFileName); $this->sitemapFile = fopen($sitemapFileName, 'w') or exit('Could not open sitemap file for writing'); @@ -85,13 +85,13 @@ private function indexChildPages($parentId) foreach ($childPages as $page) { $this->writeDebug(sprintf(" * Page \e[1m%s\e[0m (id: %d)", $page->url, $page->id)); - if (!isset($page->template->id)) { + if (! isset($page->template->id)) { $this->writeDebug(" skipping, no template found\n"); continue; } - if (!isset($page->template->properties->searchable) || $page->template->properties->searchable == 0) { + if (! isset($page->template->properties->searchable) || $page->template->properties->searchable == 0) { $this->writeDebug(" skipping, template not searchable\n"); continue; @@ -144,7 +144,7 @@ private function updatePage($menu, $lang) $customValues = []; $class = $menu->template->filename ?? ''; - $className = 'App\Http\Controllers\Page\\' . $class . 'Controller'; + $className = 'App\Http\Controllers\Page\\'.$class.'Controller'; $c = null; $priority = 1; if (class_exists($className)) { @@ -158,7 +158,7 @@ private function updatePage($menu, $lang) } $searchText = rtrim($searchText, ', '); - if (!empty($title) && !empty($searchText)) { + if (! empty($title) && ! empty($searchText)) { $result = $this->searchServer->upsertUrl($url, $title, $searchText, 'page', $lang->url, $customValues, $priority); if ($result->errorCode == 0) { @@ -183,7 +183,7 @@ private function updatePage($menu, $lang) private function updateSubPages($menu, $lang) { $class = $menu->template->filename ?? ''; - $className = 'App\Http\Controllers\Page\\' . $class . 'Controller'; + $className = 'App\Http\Controllers\Page\\'.$class.'Controller'; $c = null; // update subPage if necessary @@ -235,8 +235,8 @@ private function updateSubitems($class, $lang) private function createFolderIfNotExists($fullFilePath) { $path_parts = pathinfo($fullFilePath); - if (!file_exists($path_parts['dirname'])) { - if (!mkdir($path_parts['dirname'])) { + if (! file_exists($path_parts['dirname'])) { + if (! mkdir($path_parts['dirname'])) { printf("\n\n### Error creating sitemap folder"); } } From 404e618a143b588f64a6793caadbbd37682626e1 Mon Sep 17 00:00:00 2001 From: Thessa Kockelkorn Date: Mon, 3 Jul 2023 11:29:25 +0200 Subject: [PATCH 14/20] style: formatting --- src/Http/Guards/OpenIDGuard.php | 1 - src/Models/Indexes/SolrIndex.php | 20 ++++++++++---------- src/Models/Indexes/SolrItem.php | 14 +++++++------- src/Services/Assets/GlobalPageService.php | 18 +++++++++--------- 4 files changed, 26 insertions(+), 27 deletions(-) diff --git a/src/Http/Guards/OpenIDGuard.php b/src/Http/Guards/OpenIDGuard.php index 5d4db4e8..21e9418a 100644 --- a/src/Http/Guards/OpenIDGuard.php +++ b/src/Http/Guards/OpenIDGuard.php @@ -8,7 +8,6 @@ use Illuminate\Support\Facades\Log; use NotFound\Framework\Models\CmsGroup; use NotFound\Framework\Providers\Auth\OpenIDUserProvider; -use NotFound\Framework\Services\Auth\Token; use NotFound\Framework\Services\Auth\TokenDecoder; class OpenIDGuard implements Guard diff --git a/src/Models/Indexes/SolrIndex.php b/src/Models/Indexes/SolrIndex.php index d868fd7b..b8092a36 100644 --- a/src/Models/Indexes/SolrIndex.php +++ b/src/Models/Indexes/SolrIndex.php @@ -83,7 +83,7 @@ public function emptyCore() $json = json_decode($result); - if (!$json || !isset($json->responseHeader) || $json->responseHeader->status !== 0) { + if (! $json || ! isset($json->responseHeader) || $json->responseHeader->status !== 0) { $this->mailQueryError($url, $result); return false; @@ -100,7 +100,7 @@ private function solrHandler() { $handler = curl_init(); curl_setopt($handler, CURLOPT_RETURNTRANSFER, true); - curl_setopt($handler, CURLOPT_USERPWD, $this->solrUser . ':' . $this->solrPass); + curl_setopt($handler, CURLOPT_USERPWD, $this->solrUser.':'.$this->solrPass); curl_setopt($handler, CURLOPT_POST, true); @@ -167,7 +167,7 @@ public function addOrUpdateItem(string $url, string $title, string $contents, st public function removeItem($url) { - if (!is_null($url)) { + if (! is_null($url)) { $curl = $this->solrHandler(); $payload = ['delete' => $url]; @@ -269,7 +269,7 @@ private function mailQueryError($query, $result) } } - public function selectItems($query, $lang = 'nl', $filter = null, $start = null, $rows = null, $extraColumns = [], $highlightLength = 50, $sortField = NULL, $sortDirection = 'desc') + public function selectItems($query, $lang = 'nl', $filter = null, $start = null, $rows = null, $extraColumns = [], $highlightLength = 50, $sortField = null, $sortDirection = 'desc') { $curl = $this->solrHandler(); $url = sprintf( @@ -290,27 +290,27 @@ public function selectItems($query, $lang = 'nl', $filter = null, $start = null, $lang ); if ($filter) { - $url .= '&fq=' . $filter; + $url .= '&fq='.$filter; } if ($start && is_int($start)) { - $url .= '&start=' . $start; + $url .= '&start='.$start; } if ($rows && is_int($rows)) { - $url .= '&rows=' . $rows; + $url .= '&rows='.$rows; } if (count($extraColumns) > 0) { } if ($sortField) { - $url .= '&sort=' . urlencode($sortField . " " . $sortDirection); + $url .= '&sort='.urlencode($sortField.' '.$sortDirection); } curl_setopt($curl, CURLOPT_URL, $url); $result = curl_exec($curl); $json = json_decode($result); $searchResults = new SolrItem($json, $query, false, $highlightLength); - if (!$searchResults->isValid()) { + if (! $searchResults->isValid()) { $this->mailQueryError($url, $result); } @@ -330,7 +330,7 @@ public function suggestItems($query, $filter = null) $result = curl_exec($curl); $json = json_decode($result); $suggestions = new SolrItem($json, $query); - if (!$suggestions->isValid()) { + if (! $suggestions->isValid()) { $this->buildSuggester(); $result = curl_exec($curl); $json = json_decode($result); diff --git a/src/Models/Indexes/SolrItem.php b/src/Models/Indexes/SolrItem.php index f6b91cec..7f14e768 100644 --- a/src/Models/Indexes/SolrItem.php +++ b/src/Models/Indexes/SolrItem.php @@ -110,7 +110,7 @@ private function parseUrl($rawurl) public function getSuggestions() { $suggestions = $this->suggestions(); - if (!$suggestions) { + if (! $suggestions) { $suggestions = $this->collations(); } @@ -149,7 +149,7 @@ public function collations() } $queries[] = [ 'term' => implode(' ', $emphasizedArray), - 'payload' => '?q=' . urlencode($querytext) . '', + 'payload' => '?q='.urlencode($querytext).'', ]; } } @@ -167,10 +167,10 @@ public function predictions() $next = next($this->predictions); if (is_string($current) && is_int($next)) { $resultList[$current]['count'] = $next; - $fullTerm = $term . ' ' . $current; + $fullTerm = $term.' '.$current; $queries[] = [ 'term' => $fullTerm, - 'payload' => '?q=' . urlencode($fullTerm) . '', + 'payload' => '?q='.urlencode($fullTerm).'', ]; } } @@ -186,13 +186,13 @@ public function spellcheckList() if (isset($this->spellcheck)) { foreach ($this->spellcheck->suggestions as $suggestion) { if (isset($suggestion->startOffset)) { - $suggest = substr($query, 0, $suggestion->startOffset) . '' . $suggestion->suggestion[0] . '' . substr($query, $suggestion->endOffset); + $suggest = substr($query, 0, $suggestion->startOffset).''.$suggestion->suggestion[0].''.substr($query, $suggestion->endOffset); $suggestTerm = preg_replace('/^([a-zA-Z])+(_[a-zA-Z]{2})?:/', '', $suggest); // remove search field if necessary - $suggest_url = substr($query, 0, $suggestion->startOffset) . $suggestion->suggestion[0] . substr($query, $suggestion->endOffset); + $suggest_url = substr($query, 0, $suggestion->startOffset).$suggestion->suggestion[0].substr($query, $suggestion->endOffset); $suggest_url = preg_replace('/^([a-zA-Z])+(_[a-zA-Z]{2})?:/', '', $suggest_url); // remove search field if necessary - $items[] = (object) ['link' => '?q=' . rawurlencode(urldecode($suggest_url)), 'text' => urldecode($suggestTerm)]; + $items[] = (object) ['link' => '?q='.rawurlencode(urldecode($suggest_url)), 'text' => urldecode($suggestTerm)]; } } } diff --git a/src/Services/Assets/GlobalPageService.php b/src/Services/Assets/GlobalPageService.php index 72a39880..dc1bdd51 100644 --- a/src/Services/Assets/GlobalPageService.php +++ b/src/Services/Assets/GlobalPageService.php @@ -27,15 +27,15 @@ public function getType(): AssetType return AssetType::PAGE; } - public function getComponents(): Collection - { - return $this->fieldComponents; - } - - protected function getCacheKey(): string - { - return $this->lang->url.'page_globals'; - } + public function getComponents(): Collection + { + return $this->fieldComponents; + } + + protected function getCacheKey(): string + { + return $this->lang->url.'page_globals'; + } /** * Loops through all the table items and return them with the appropriate Input Class From 4c7c9321d2236605002186bc5241742898384ae8 Mon Sep 17 00:00:00 2001 From: Thessa Kockelkorn Date: Mon, 3 Jul 2023 12:37:15 +0200 Subject: [PATCH 15/20] fix: writing sitemap --- src/Services/Indexer/IndexBuilderService.php | 122 ++++++++++--------- 1 file changed, 66 insertions(+), 56 deletions(-) diff --git a/src/Services/Indexer/IndexBuilderService.php b/src/Services/Indexer/IndexBuilderService.php index 5d769117..95b6d1b8 100644 --- a/src/Services/Indexer/IndexBuilderService.php +++ b/src/Services/Indexer/IndexBuilderService.php @@ -13,9 +13,8 @@ class IndexBuilderService private $locales; - private $defaultLocale; - private $domainName; + private $domain; private $sitemapFile; @@ -28,7 +27,7 @@ public function __construct(string $serverType, $debug = false) $locale = env('SB_LOCALES_DEFAULT', 'nl'); $this->defaultLocale = Lang::where('url', $locale)->get(); - $this->domainName = env('APP_NAME'); + $this->domain = rtrim(env('APP_URL', ""), "/"); switch ($serverType) { case 'solr': $this->searchServer = new SolrIndexService($this->debug); @@ -44,7 +43,7 @@ public function run() if (count($sites) > 0) { $startResult = $this->searchServer->startUpdate(); - if (! $startResult) { + if (!$startResult) { $this->writeDebug("\n\n Error when emptying core! \n\n"); } @@ -85,13 +84,13 @@ private function indexChildPages($parentId) foreach ($childPages as $page) { $this->writeDebug(sprintf(" * Page \e[1m%s\e[0m (id: %d)", $page->url, $page->id)); - if (! isset($page->template->id)) { + if (!isset($page->template->id)) { $this->writeDebug(" skipping, no template found\n"); continue; } - if (! isset($page->template->properties->searchable) || $page->template->properties->searchable == 0) { + if (!isset($page->template->properties->searchable) || $page->template->properties->searchable == 0) { $this->writeDebug(" skipping, template not searchable\n"); continue; @@ -104,15 +103,11 @@ private function indexChildPages($parentId) $menu = Menu::whereId($page->id)->firstOrFail(); - if ($this->searchServer->urlNeedsUpdate($menu->getPath(), strtotime($menu->updated_at))) { - $this->writeDebug(': update needed: '); - foreach ($this->locales as $lang) { - $this->updatePage($menu, $lang); - } - } else { - $this->writeDebug(": Does not need updating\n"); + foreach ($this->locales as $lang) { + $this->updatePage($menu, $lang); } + // index subitems for page foreach ($this->locales as $lang) { $this->updateSubPages($menu, $lang); @@ -124,66 +119,72 @@ private function indexChildPages($parentId) private function updatePage($menu, $lang) { - $success = true; app()->setLocale($lang->url); - - if ($this->sitemapFile) { - $sitemap = ''; - } - $searchText = ''; - $pageService = new PageService($menu, $lang); - $title = $menu->getTitle($lang); if (count($this->locales) == 1) { $url = $menu->getPath(); } else { $url = $menu->getLocalizedPath(); } - $searchText = $pageService->getContentForIndexer(); - // continue with customValues - $customValues = []; + if ($this->searchServer->urlNeedsUpdate($menu->getPath(), strtotime($menu->updated_at))) { + $this->writeDebug(': update needed: '); - $class = $menu->template->filename ?? ''; - $className = 'App\Http\Controllers\Page\\'.$class.'Controller'; - $c = null; - $priority = 1; - if (class_exists($className)) { - $c = new $className(); - if (method_exists($className, 'customSearchValues')) { - $customValues = $c->customSearchValues($menu->id); - } - if (method_exists($className, 'searchPriority')) { - $priority = $c->searchPriority(); - } - } - $searchText = rtrim($searchText, ', '); - if (! empty($title) && ! empty($searchText)) { - $result = $this->searchServer->upsertUrl($url, $title, $searchText, 'page', $lang->url, $customValues, $priority); - if ($result->errorCode == 0) { - $this->writeDebug(" success\n"); - } else { - $this->writeDebug(" FAILED\n"); + $searchText = ''; + $pageService = new PageService($menu, $lang); + $title = $menu->getTitle($lang); + + $searchText = $pageService->getContentForIndexer(); + + // continue with customValues + $customValues = []; + + $class = $menu->template->filename ?? ''; + $className = 'App\Http\Controllers\Page\\' . $class . 'Controller'; + $c = null; + $priority = 1; + if (class_exists($className)) { + $c = new $className(); + if (method_exists($className, 'customSearchValues')) { + $customValues = $c->customSearchValues($menu->id); + } + if (method_exists($className, 'searchPriority')) { + $priority = $c->searchPriority(); + } } - if ($this->sitemapFile) { - // update sitemap - $sitemap .= sprintf( - "%s%s\r\n", - $this->domainName, - $url - ); + $searchText = rtrim($searchText, ', '); + if (!empty($title) && !empty($searchText)) { + $result = $this->searchServer->upsertUrl($url, $title, $searchText, 'page', $lang->url, $customValues, $priority); + + if ($result->errorCode == 0) { + $this->writeDebug(" success\n"); + } else { + $this->writeDebug(" FAILED\n"); + } + } else { + $this->writeDebug(" empty page or title\n"); } } else { - $this->writeDebug(" empty page or title\n"); + $this->writeDebug(": Does not need updating\n"); + } + + if ($this->sitemapFile) { + // update sitemap + $sitemap = sprintf( + "%s%s\r\n", + $this->domain, + $url + ); + fwrite($this->sitemapFile, $sitemap); } } private function updateSubPages($menu, $lang) { $class = $menu->template->filename ?? ''; - $className = 'App\Http\Controllers\Page\\'.$class.'Controller'; + $className = 'App\Http\Controllers\Page\\' . $class . 'Controller'; $c = null; // update subPage if necessary @@ -215,7 +216,7 @@ private function updateSubitems($class, $lang) if ($this->sitemapFile && $searchItem['sitemap']) { $sitemap = sprintf( "%s%s\r\n", - $this->domainName, + $this->domain, $url ); } @@ -228,6 +229,15 @@ private function updateSubitems($class, $lang) } else { $this->writeDebug(": Does not need updating\n"); } + + if ($this->sitemapFile) { + $sitemap = sprintf( + "%s%s\r\n", + $this->domain, + $url + ); + fwrite($this->sitemapFile, $sitemap); + } } } } @@ -235,8 +245,8 @@ private function updateSubitems($class, $lang) private function createFolderIfNotExists($fullFilePath) { $path_parts = pathinfo($fullFilePath); - if (! file_exists($path_parts['dirname'])) { - if (! mkdir($path_parts['dirname'])) { + if (!file_exists($path_parts['dirname'])) { + if (!mkdir($path_parts['dirname'])) { printf("\n\n### Error creating sitemap folder"); } } From 9a9f1f43895984184fa65c4319c8d44dcac07185 Mon Sep 17 00:00:00 2001 From: Thessa Kockelkorn Date: Mon, 3 Jul 2023 13:51:30 +0200 Subject: [PATCH 16/20] fix: replace html tags by space --- src/Models/Indexes/SolrIndex.php | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Models/Indexes/SolrIndex.php b/src/Models/Indexes/SolrIndex.php index b8092a36..37521ae3 100644 --- a/src/Models/Indexes/SolrIndex.php +++ b/src/Models/Indexes/SolrIndex.php @@ -83,7 +83,7 @@ public function emptyCore() $json = json_decode($result); - if (! $json || ! isset($json->responseHeader) || $json->responseHeader->status !== 0) { + if (!$json || !isset($json->responseHeader) || $json->responseHeader->status !== 0) { $this->mailQueryError($url, $result); return false; @@ -100,7 +100,7 @@ private function solrHandler() { $handler = curl_init(); curl_setopt($handler, CURLOPT_RETURNTRANSFER, true); - curl_setopt($handler, CURLOPT_USERPWD, $this->solrUser.':'.$this->solrPass); + curl_setopt($handler, CURLOPT_USERPWD, $this->solrUser . ':' . $this->solrPass); curl_setopt($handler, CURLOPT_POST, true); @@ -135,7 +135,7 @@ public function addOrUpdateItem(string $url, string $title, string $contents, st $curl = $this->solrHandler(); $doc = [ sprintf('title_%s', $lang) => $title, - sprintf('content_%s', $lang) => html_entity_decode(trim(preg_replace('/\s+/', ' ', strip_tags($contents)))), + sprintf('content_%s', $lang) => html_entity_decode(trim(preg_replace('/\s+/', ' ', preg_replace('#<[^>]+>#', ' ', $contents)))), 'type' => $type, 'url' => $url, 'priority' => $priority, @@ -167,7 +167,7 @@ public function addOrUpdateItem(string $url, string $title, string $contents, st public function removeItem($url) { - if (! is_null($url)) { + if (!is_null($url)) { $curl = $this->solrHandler(); $payload = ['delete' => $url]; @@ -290,27 +290,27 @@ public function selectItems($query, $lang = 'nl', $filter = null, $start = null, $lang ); if ($filter) { - $url .= '&fq='.$filter; + $url .= '&fq=' . $filter; } if ($start && is_int($start)) { - $url .= '&start='.$start; + $url .= '&start=' . $start; } if ($rows && is_int($rows)) { - $url .= '&rows='.$rows; + $url .= '&rows=' . $rows; } if (count($extraColumns) > 0) { } if ($sortField) { - $url .= '&sort='.urlencode($sortField.' '.$sortDirection); + $url .= '&sort=' . urlencode($sortField . ' ' . $sortDirection); } curl_setopt($curl, CURLOPT_URL, $url); $result = curl_exec($curl); $json = json_decode($result); $searchResults = new SolrItem($json, $query, false, $highlightLength); - if (! $searchResults->isValid()) { + if (!$searchResults->isValid()) { $this->mailQueryError($url, $result); } @@ -330,7 +330,7 @@ public function suggestItems($query, $filter = null) $result = curl_exec($curl); $json = json_decode($result); $suggestions = new SolrItem($json, $query); - if (! $suggestions->isValid()) { + if (!$suggestions->isValid()) { $this->buildSuggester(); $result = curl_exec($curl); $json = json_decode($result); From 59fec8a8d03fa7cd4c44b0317ebc044c29b1a4fe Mon Sep 17 00:00:00 2001 From: Thessa Kockelkorn Date: Mon, 3 Jul 2023 13:59:43 +0200 Subject: [PATCH 17/20] style: formatting --- routes/api.php | 21 +++++++++---------- src/Models/Indexes/SolrIndex.php | 18 ++++++++-------- src/Services/Indexer/IndexBuilderService.php | 22 ++++++++------------ 3 files changed, 28 insertions(+), 33 deletions(-) diff --git a/routes/api.php b/routes/api.php index 63591b50..6c380f83 100644 --- a/routes/api.php +++ b/routes/api.php @@ -10,7 +10,6 @@ use NotFound\Framework\Http\Controllers\SettingsController; use NotFound\Framework\Http\Controllers\Support\SupportController; use NotFound\Framework\Http\Controllers\UserPreferencesController; -use Siteboss\Routes\SiteRoutes; use Spatie\Honeypot\ProtectAgainstSpam; // ContentBlock @@ -51,23 +50,23 @@ Route::get('contentblocks/{csvTables}', [ContentBlockController::class, 'get']); // Table editor - Route::prefix('table')->group(__DIR__ . '/cms/table.php'); + Route::prefix('table')->group(__DIR__.'/cms/table.php'); // Form builder - Route::middleware('role:forms')->group(__DIR__ . '/cms/forms.php'); + Route::middleware('role:forms')->group(__DIR__.'/cms/forms.php'); Route::prefix('app')->group(function () { - if (file_exists(base_path() . '/routes/siteboss.php')) { + if (file_exists(base_path().'/routes/siteboss.php')) { Route::prefix('site')->group( - base_path() . '/routes/siteboss.php' + base_path().'/routes/siteboss.php' ); } // /menu - Route::prefix('menu')->group(__DIR__ . '/cms/menu.php'); + Route::prefix('menu')->group(__DIR__.'/cms/menu.php'); // Page editor - Route::prefix('page')->group(__DIR__ . '/cms/page.php'); + Route::prefix('page')->group(__DIR__.'/cms/page.php'); // Settings Route::get('settings', [SettingsController::class, 'index']); @@ -78,10 +77,10 @@ Route::get('preferences', [UserPreferencesController::class, 'index']); Route::post('preferences', [UserPreferencesController::class, 'update']); - Route::prefix('users')->group(__DIR__ . '/cms/users.php'); + Route::prefix('users')->group(__DIR__.'/cms/users.php'); // CMS Editor - Route::prefix('editor')->group(__DIR__ . '/cms/editor.php'); + Route::prefix('editor')->group(__DIR__.'/cms/editor.php'); // About SiteBoss CMS page Route::get('about', [AboutController::class, 'index']); @@ -93,10 +92,10 @@ }); // AutoLayout Demo pages - Route::prefix('demo')->group(__DIR__ . '/cms/demo.php'); + Route::prefix('demo')->group(__DIR__.'/cms/demo.php'); // Domain forward manager - Route::prefix('forwards')->group(__DIR__ . '/cms/forwards.php'); + Route::prefix('forwards')->group(__DIR__.'/cms/forwards.php'); }); }); }); diff --git a/src/Models/Indexes/SolrIndex.php b/src/Models/Indexes/SolrIndex.php index 37521ae3..3fdb4264 100644 --- a/src/Models/Indexes/SolrIndex.php +++ b/src/Models/Indexes/SolrIndex.php @@ -83,7 +83,7 @@ public function emptyCore() $json = json_decode($result); - if (!$json || !isset($json->responseHeader) || $json->responseHeader->status !== 0) { + if (! $json || ! isset($json->responseHeader) || $json->responseHeader->status !== 0) { $this->mailQueryError($url, $result); return false; @@ -100,7 +100,7 @@ private function solrHandler() { $handler = curl_init(); curl_setopt($handler, CURLOPT_RETURNTRANSFER, true); - curl_setopt($handler, CURLOPT_USERPWD, $this->solrUser . ':' . $this->solrPass); + curl_setopt($handler, CURLOPT_USERPWD, $this->solrUser.':'.$this->solrPass); curl_setopt($handler, CURLOPT_POST, true); @@ -167,7 +167,7 @@ public function addOrUpdateItem(string $url, string $title, string $contents, st public function removeItem($url) { - if (!is_null($url)) { + if (! is_null($url)) { $curl = $this->solrHandler(); $payload = ['delete' => $url]; @@ -290,27 +290,27 @@ public function selectItems($query, $lang = 'nl', $filter = null, $start = null, $lang ); if ($filter) { - $url .= '&fq=' . $filter; + $url .= '&fq='.$filter; } if ($start && is_int($start)) { - $url .= '&start=' . $start; + $url .= '&start='.$start; } if ($rows && is_int($rows)) { - $url .= '&rows=' . $rows; + $url .= '&rows='.$rows; } if (count($extraColumns) > 0) { } if ($sortField) { - $url .= '&sort=' . urlencode($sortField . ' ' . $sortDirection); + $url .= '&sort='.urlencode($sortField.' '.$sortDirection); } curl_setopt($curl, CURLOPT_URL, $url); $result = curl_exec($curl); $json = json_decode($result); $searchResults = new SolrItem($json, $query, false, $highlightLength); - if (!$searchResults->isValid()) { + if (! $searchResults->isValid()) { $this->mailQueryError($url, $result); } @@ -330,7 +330,7 @@ public function suggestItems($query, $filter = null) $result = curl_exec($curl); $json = json_decode($result); $suggestions = new SolrItem($json, $query); - if (!$suggestions->isValid()) { + if (! $suggestions->isValid()) { $this->buildSuggester(); $result = curl_exec($curl); $json = json_decode($result); diff --git a/src/Services/Indexer/IndexBuilderService.php b/src/Services/Indexer/IndexBuilderService.php index 95b6d1b8..dd6edb83 100644 --- a/src/Services/Indexer/IndexBuilderService.php +++ b/src/Services/Indexer/IndexBuilderService.php @@ -13,7 +13,6 @@ class IndexBuilderService private $locales; - private $domain; private $sitemapFile; @@ -27,7 +26,7 @@ public function __construct(string $serverType, $debug = false) $locale = env('SB_LOCALES_DEFAULT', 'nl'); $this->defaultLocale = Lang::where('url', $locale)->get(); - $this->domain = rtrim(env('APP_URL', ""), "/"); + $this->domain = rtrim(env('APP_URL', ''), '/'); switch ($serverType) { case 'solr': $this->searchServer = new SolrIndexService($this->debug); @@ -43,7 +42,7 @@ public function run() if (count($sites) > 0) { $startResult = $this->searchServer->startUpdate(); - if (!$startResult) { + if (! $startResult) { $this->writeDebug("\n\n Error when emptying core! \n\n"); } @@ -84,13 +83,13 @@ private function indexChildPages($parentId) foreach ($childPages as $page) { $this->writeDebug(sprintf(" * Page \e[1m%s\e[0m (id: %d)", $page->url, $page->id)); - if (!isset($page->template->id)) { + if (! isset($page->template->id)) { $this->writeDebug(" skipping, no template found\n"); continue; } - if (!isset($page->template->properties->searchable) || $page->template->properties->searchable == 0) { + if (! isset($page->template->properties->searchable) || $page->template->properties->searchable == 0) { $this->writeDebug(" skipping, template not searchable\n"); continue; @@ -103,7 +102,6 @@ private function indexChildPages($parentId) $menu = Menu::whereId($page->id)->firstOrFail(); - foreach ($this->locales as $lang) { $this->updatePage($menu, $lang); } @@ -129,8 +127,6 @@ private function updatePage($menu, $lang) if ($this->searchServer->urlNeedsUpdate($menu->getPath(), strtotime($menu->updated_at))) { $this->writeDebug(': update needed: '); - - $searchText = ''; $pageService = new PageService($menu, $lang); $title = $menu->getTitle($lang); @@ -141,7 +137,7 @@ private function updatePage($menu, $lang) $customValues = []; $class = $menu->template->filename ?? ''; - $className = 'App\Http\Controllers\Page\\' . $class . 'Controller'; + $className = 'App\Http\Controllers\Page\\'.$class.'Controller'; $c = null; $priority = 1; if (class_exists($className)) { @@ -155,7 +151,7 @@ private function updatePage($menu, $lang) } $searchText = rtrim($searchText, ', '); - if (!empty($title) && !empty($searchText)) { + if (! empty($title) && ! empty($searchText)) { $result = $this->searchServer->upsertUrl($url, $title, $searchText, 'page', $lang->url, $customValues, $priority); if ($result->errorCode == 0) { @@ -184,7 +180,7 @@ private function updatePage($menu, $lang) private function updateSubPages($menu, $lang) { $class = $menu->template->filename ?? ''; - $className = 'App\Http\Controllers\Page\\' . $class . 'Controller'; + $className = 'App\Http\Controllers\Page\\'.$class.'Controller'; $c = null; // update subPage if necessary @@ -245,8 +241,8 @@ private function updateSubitems($class, $lang) private function createFolderIfNotExists($fullFilePath) { $path_parts = pathinfo($fullFilePath); - if (!file_exists($path_parts['dirname'])) { - if (!mkdir($path_parts['dirname'])) { + if (! file_exists($path_parts['dirname'])) { + if (! mkdir($path_parts['dirname'])) { printf("\n\n### Error creating sitemap folder"); } } From 7e709cf03988592aeda0a3a15446375fd00dcbec Mon Sep 17 00:00:00 2001 From: Rene Date: Mon, 3 Jul 2023 16:25:42 +0200 Subject: [PATCH 18/20] feat: error when solr hostname cannot be resolved --- src/Models/Indexes/SolrIndex.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Models/Indexes/SolrIndex.php b/src/Models/Indexes/SolrIndex.php index 3fdb4264..facd3733 100644 --- a/src/Models/Indexes/SolrIndex.php +++ b/src/Models/Indexes/SolrIndex.php @@ -157,6 +157,10 @@ public function addOrUpdateItem(string $url, string $title, string $contents, st $result = curl_exec($curl); + if (curl_errno($curl) === 6) { + exit('[ERROR] Could not resolve solr host: '.$this->getSolrBaseUrl()); + } + $json = json_decode($result); if ($json && isset($json->responseHeader) && $json->responseHeader->status == 0) { return true; From 3153f157aa5b7ae68d5cc95a49096f91d8e92b4c Mon Sep 17 00:00:00 2001 From: Thessa Kockelkorn Date: Tue, 4 Jul 2023 10:41:34 +0200 Subject: [PATCH 19/20] fix: remove unnecessary code --- src/Services/Indexer/IndexBuilderService.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Services/Indexer/IndexBuilderService.php b/src/Services/Indexer/IndexBuilderService.php index dd6edb83..b819e025 100644 --- a/src/Services/Indexer/IndexBuilderService.php +++ b/src/Services/Indexer/IndexBuilderService.php @@ -24,8 +24,6 @@ public function __construct(string $serverType, $debug = false) $this->debug = $debug; $this->locales = Lang::all(); - $locale = env('SB_LOCALES_DEFAULT', 'nl'); - $this->defaultLocale = Lang::where('url', $locale)->get(); $this->domain = rtrim(env('APP_URL', ''), '/'); switch ($serverType) { case 'solr': From 3f360b82135dc8b08b754123383ebdeb86034537 Mon Sep 17 00:00:00 2001 From: Thessa Kockelkorn Date: Tue, 4 Jul 2023 11:41:10 +0200 Subject: [PATCH 20/20] fix: pass localized url --- src/Services/Indexer/IndexBuilderService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Services/Indexer/IndexBuilderService.php b/src/Services/Indexer/IndexBuilderService.php index b819e025..5e70796f 100644 --- a/src/Services/Indexer/IndexBuilderService.php +++ b/src/Services/Indexer/IndexBuilderService.php @@ -122,7 +122,7 @@ private function updatePage($menu, $lang) $url = $menu->getLocalizedPath(); } - if ($this->searchServer->urlNeedsUpdate($menu->getPath(), strtotime($menu->updated_at))) { + if ($this->searchServer->urlNeedsUpdate($url, strtotime($menu->updated_at))) { $this->writeDebug(': update needed: '); $searchText = '';