From 0a2163119de9f4313773e3bca9e498b51beaaa84 Mon Sep 17 00:00:00 2001 From: Xander Schuurman <44030544+keeama13@users.noreply.github.com> Date: Mon, 26 Jun 2023 12:48:37 +0200 Subject: [PATCH 01/13] feat: added spatie honeypot (#34) --- composer.json | 1 + resources/views/siteboss/forms/form.blade.php | 1 + routes/api.php | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 7c16ab37..041df2fa 100644 --- a/composer.json +++ b/composer.json @@ -17,6 +17,7 @@ ], "require": { "spatie/laravel-package-tools": "^1.14.0", + "spatie/laravel-honeypot": "^4.3.2", "illuminate/contracts": "^10.0", "notfoundnl/siteboss-layout": "^1.3.0", "mcamara/laravel-localization": "^1.8", diff --git a/resources/views/siteboss/forms/form.blade.php b/resources/views/siteboss/forms/form.blade.php index 39f7d247..47b4980e 100644 --- a/resources/views/siteboss/forms/form.blade.php +++ b/resources/views/siteboss/forms/form.blade.php @@ -5,6 +5,7 @@ enctype="multipart/form-data" > +
@foreach ($form->fields as $item)
diff --git a/routes/api.php b/routes/api.php index 1a993802..fb3745b6 100644 --- a/routes/api.php +++ b/routes/api.php @@ -10,6 +10,7 @@ use NotFound\Framework\Http\Controllers\SettingsController; use NotFound\Framework\Http\Controllers\Support\SupportController; use NotFound\Framework\Http\Controllers\UserPreferencesController; +use Spatie\Honeypot\ProtectAgainstSpam; // ContentBlock /* @@ -26,7 +27,7 @@ // Unauthenticated routes Route::prefix('api')->group(function () { Route::namespace('Forms')->group(function () { - Route::post('forms/{form:id}/{langurl}', [DataController::class, 'create'])->name('formbuilder.post'); + Route::post('forms/{form:id}/{langurl}', [DataController::class, 'create'])->middleware(ProtectAgainstSpam::class)->name('formbuilder.post'); Route::get('fields/{id}', [FieldController::class, 'readOneJson']); // RIGHTS!!!!! Route::get('download/{submitid}/{fieldId}/{UUID}', [DownloadController::class, 'unauthenticatedDownload']); From 75d53319a1ec269b86d55ef660b089eb4ffb2d76 Mon Sep 17 00:00:00 2001 From: Xander Schuurman <44030544+keeama13@users.noreply.github.com> Date: Tue, 27 Jun 2023 11:54:56 +0200 Subject: [PATCH 02/13] fix: datacontroller change (#36) --- src/Http/Controllers/Forms/DataController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Http/Controllers/Forms/DataController.php b/src/Http/Controllers/Forms/DataController.php index 8ee61eff..d7ac887a 100644 --- a/src/Http/Controllers/Forms/DataController.php +++ b/src/Http/Controllers/Forms/DataController.php @@ -127,7 +127,7 @@ public function deleteRow($formId, $recordId) private function runSuccessAction($langurl, $formInfo, $formValidator) { // Trigger default form handler - $action = SitebossHelper::config('form_success_action') ?? ''; + $action = SitebossHelper::config('form_success_action', false) ?? ''; if ($action && trim($action !== '')) { $actionClass = new $action($langurl, $formInfo, $formValidator); $actionClass->run(); From b934e512bdd9ac50ae0ddebc40e89f84fc0c7e52 Mon Sep 17 00:00:00 2001 From: Xander Schuurman <44030544+keeama13@users.noreply.github.com> Date: Tue, 27 Jun 2023 13:36:08 +0200 Subject: [PATCH 03/13] feat: custom spam response (#37) * feat: custom spam response * fix: typo --- config/honeypot.php | 68 +++++++++++++++++++++ src/SpamResponder/SpamDetectedResponder.php | 15 +++++ 2 files changed, 83 insertions(+) create mode 100644 config/honeypot.php create mode 100644 src/SpamResponder/SpamDetectedResponder.php diff --git a/config/honeypot.php b/config/honeypot.php new file mode 100644 index 00000000..e1ab6f81 --- /dev/null +++ b/config/honeypot.php @@ -0,0 +1,68 @@ + env('HONEYPOT_ENABLED', true), + + /* + * Here you can specify name of the honeypot field. Any requests that submit a non-empty + * value for this name will be discarded. Make sure this name does not + * collide with a form field that is actually used. + */ + 'name_field_name' => env('HONEYPOT_NAME', 'my_name'), + + /* + * When this is activated there will be a random string added + * to the name_field_name. This improves the + * protection against bots. + */ + 'randomize_name_field_name' => env('HONEYPOT_RANDOMIZE', true), + + /* + * When this is activated, requests will be checked if + * form is submitted faster than this amount of seconds + */ + 'valid_from_timestamp' => env('HONEYPOT_VALID_FROM_TIMESTAMP', true), + + /* + * This field contains the name of a form field that will be used to verify + * if the form wasn't submitted too quickly. Make sure this name does not + * collide with a form field that is actually used. + */ + 'valid_from_field_name' => env('HONEYPOT_VALID_FROM', 'valid_from'), + + /* + * If the form is submitted faster than this amount of seconds + * the form submission will be considered invalid. + */ + 'amount_of_seconds' => env('HONEYPOT_SECONDS', 3), + + /* + * This class is responsible for sending a response to requests that + * are detected as being spammy. By default a blank page is shown. + * + * A valid responder is any class that implements + * `Spatie\Honeypot\SpamResponder\SpamResponder` + */ + 'respond_to_spam_with' => SpamDetectedResponder::class, + + /* + * When activated, requests will be checked if honeypot fields are missing, + * if so the request will be stamped as spam. Be careful! When using the + * global middleware be sure to add honeypot fields to each form. + */ + 'honeypot_fields_required_for_all_forms' => false, + + /* + * This class is responsible for applying all spam protection + * rules for a request. In most cases, you shouldn't change + * this value. + */ + 'spam_protection' => \Spatie\Honeypot\SpamProtection::class, +]; diff --git a/src/SpamResponder/SpamDetectedResponder.php b/src/SpamResponder/SpamDetectedResponder.php new file mode 100644 index 00000000..6d1fe12c --- /dev/null +++ b/src/SpamResponder/SpamDetectedResponder.php @@ -0,0 +1,15 @@ + Date: Tue, 27 Jun 2023 16:07:09 +0200 Subject: [PATCH 04/13] feat: honeypot serviceprovider (#38) --- src/FrameworkServiceProvider.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/FrameworkServiceProvider.php b/src/FrameworkServiceProvider.php index f91a72b3..feafe3af 100644 --- a/src/FrameworkServiceProvider.php +++ b/src/FrameworkServiceProvider.php @@ -21,6 +21,7 @@ public function boot(): void $this->publishes([ __DIR__.'/../config/app.php' => config_path('app.php'), __DIR__.'/../config/auth.php' => config_path('auth.php'), + __DIR__.'/../config/honeypot.php' => config_path('honeypot.php'), __DIR__.'/../config/siteboss.php' => config_path('siteboss.php'), __DIR__.'/../config/openid.php' => config_path('openid.php'), __DIR__.'/../config/clamav.php' => config_path('clamav.php'), From 6dc9b8b6411fd5818c153c4e7ae8d88fe4c0d26a Mon Sep 17 00:00:00 2001 From: Rene Date: Tue, 27 Jun 2023 16:45:53 +0200 Subject: [PATCH 05/13] fix: better error for unsupported locales --- src/Models/Lang.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Models/Lang.php b/src/Models/Lang.php index b1d2efe8..35e8b957 100644 --- a/src/Models/Lang.php +++ b/src/Models/Lang.php @@ -68,7 +68,10 @@ public static function current() { if (is_null(self::$current)) { $locale = app()->getLocale(); - self::$current = self::query()->whereUrl($locale)->firstOrFail(); + self::$current = self::query()->whereUrl($locale)->first(); + if (! isset(self::$current->id)) { + exit('Locale not supported: '.$locale."\nCheck config/app.php and your database\n"); + } } return self::$current; From 3d41f3ff7c391dbf7399a8dcc2e559afba9b127f Mon Sep 17 00:00:00 2001 From: Rene Date: Thu, 29 Jun 2023 11:29:56 +0200 Subject: [PATCH 06/13] feat: debug message for file permissions --- src/Services/Assets/Components/ComponentImage.php | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Services/Assets/Components/ComponentImage.php b/src/Services/Assets/Components/ComponentImage.php index 45e4dbe6..a5f2b9e5 100644 --- a/src/Services/Assets/Components/ComponentImage.php +++ b/src/Services/Assets/Components/ComponentImage.php @@ -209,9 +209,17 @@ private function relativePathToPublicDisk(): string */ private function makeParentDirectories(): bool { - return make_directories( + $createDirs = make_directories( Storage::path('public'), $this->subFolderPublic.$this->assetModel->getIdentifier().'/'.$this->assetItem->internal.'/' ); + + // if app is running in debug mode, throw an error if the directories could not be created + if (env('APP_DEBUG') === true && ! $createDirs) { + exit('Could not create directories'); + } + + return $createDirs; + } } From e8e36476b5a17739295ac886be867ce30167291d Mon Sep 17 00:00:00 2001 From: Rene Date: Mon, 3 Jul 2023 12:15:36 +0200 Subject: [PATCH 07/13] style: formatting --- src/Http/Guards/OpenIDGuard.php | 1 - src/Services/Assets/GlobalPageService.php | 18 +++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/Http/Guards/OpenIDGuard.php b/src/Http/Guards/OpenIDGuard.php index 5d4db4e8..21e9418a 100644 --- a/src/Http/Guards/OpenIDGuard.php +++ b/src/Http/Guards/OpenIDGuard.php @@ -8,7 +8,6 @@ use Illuminate\Support\Facades\Log; use NotFound\Framework\Models\CmsGroup; use NotFound\Framework\Providers\Auth\OpenIDUserProvider; -use NotFound\Framework\Services\Auth\Token; use NotFound\Framework\Services\Auth\TokenDecoder; class OpenIDGuard implements Guard diff --git a/src/Services/Assets/GlobalPageService.php b/src/Services/Assets/GlobalPageService.php index 72a39880..dc1bdd51 100644 --- a/src/Services/Assets/GlobalPageService.php +++ b/src/Services/Assets/GlobalPageService.php @@ -27,15 +27,15 @@ public function getType(): AssetType return AssetType::PAGE; } - public function getComponents(): Collection - { - return $this->fieldComponents; - } - - protected function getCacheKey(): string - { - return $this->lang->url.'page_globals'; - } + public function getComponents(): Collection + { + return $this->fieldComponents; + } + + protected function getCacheKey(): string + { + return $this->lang->url.'page_globals'; + } /** * Loops through all the table items and return them with the appropriate Input Class From af1e27d4c705194334f41480a78ec5e6baf32bc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9?= Date: Mon, 3 Jul 2023 12:44:21 +0200 Subject: [PATCH 08/13] feat: Update issue templates (#41) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Version 2.0.0 (#26) * feat: packaged cmseditor (#2) * feat: composer file * feat: add license + changelog * Add files via upload * Delete CmsEditor directory * Delete resources/views directory --------- Co-authored-by: Rene Co-authored-by: Xander Schuurman <44030544+keeama13@users.noreply.github.com> * feat: migrations + routes + restructure (#3) * feat: migrations + routes + restructure * fix: added composer + removed cmseditor folder * fix: providers * feat: models in package (#4) * feat: models in package * feat: packaged controllers * feat: seeders in package * fix: removed test echo * feat: lang + views * feat: services + provides * feat: package * feat: finalizing package * feat: last changes * fix: remove surplus migrations * feat: database seeders * feat: add helpers * fix: policy namespacing * fix: class name * fix: remove user provider * feat: last configs --------- Co-authored-by: Rene * fix: syntax error in composer file * style: formatting * feat: add pinter * fix!: database prefix (#6) * feat: Events (#7) * feat: models in package * feat: packaged controllers * feat: seeders in package * fix: removed test echo * feat: lang + views * feat: services + provides * feat: package * feat: finalizing package * feat: last changes * fix: remove surplus migrations * feat: database seeders * feat: add helpers * fix: policy namespacing * fix: class name * fix: remove user provider * feat: last configs * feat: events * feat: events namespace * fix: namespace errors * style: formatting * fix: composer syntax error --------- Co-authored-by: keeama13 Co-authored-by: Rene * fix: forms * feat: namespace changes (#9) * feat: namespace changes * feat: namespacing * feat: working views * fix: remove duplicate entries * fix: remove SiteBoss namspace * style: formatting --------- Co-authored-by: René Co-authored-by: Rene * fix: removed laravel assets from publish (#10) * fix: class path name * fix!: formbuilder problem quickfix * style: formatting * Fix/solr indexer (#13) * fix: add timestamps to search table * fix: replace all white spaces by single spaces * fix: get page title * style: formatting --------- Co-authored-by: Thessa Kockelkorn Co-authored-by: René * feat: add Siteboss Helper * fix: helper class namespace * fix: visible for CmsConfig * feat: remove unused function (#15) * feat: removed siteboss_path() * feat: app config, routes, factory updated * fix: factorytype * style: formatting * style: formatting * feat: SOLR localization (#14) * feat: forced localization of path * fix: mv creation of content string to pageservice * fix: get localized title --------- Co-authored-by: Thessa Kockelkorn * feat: searchable * feat!: Form handler (#16) * fix: clamav no socket * feat: call FormHandler * fix: re-add getSummaryHtml * fix getSummaryHtml for files * style: formatting --------- Co-authored-by: René * feat: mail helper (#17) * feat: send mail from helper * style: formatting * fix: send debug mails --------- Co-authored-by: Rene * fix: mail forms (#19) * fix: mailhandler (#21) * fix: mailhandler * fix: namespacing * fix: mail views * fix: mail view from package * style: formatting * feat: add todo --------- Co-authored-by: Rene * fix: move ContentBlockService (#23) * fix: move ContentBlockService * style: formatting --------- Co-authored-by: Thessa Kockelkorn Co-authored-by: Rene * feat: use noIndex server property (#22) * feat: use noIndex server property * style: formatting --------- Co-authored-by: Thessa Kockelkorn Co-authored-by: Rene * fix: handle empty notification_address * fix: router no longer ignores enabled * feat: TableSelect support for translated table * Fix: pick laravel old (#24) * fix: slug trim (#309) * fix: validate error on document update * fix: slug trim --------- Co-authored-by: René * fix: date not required (#312) * fix: changed empty date to null value * style: formatting * fix: perserve params in table editor * fix: use params in breadcrumb * feat: title search * fix: Moved overview setting to correct properties (#314) * chore: rename sample configuration * fix: ordering * fix: date localized display * feat: check database (#315) * feat: column checks * fix: implementation for repeatable * fix: localized fields --------- Co-authored-by: Rene * feat: timepicker (#313) * fix: date can be empty in overview * fix: paging * fix: columntype for button * feat: add Laravel's email verification middleware (#311) * feat: add Laravel's email verification middleware * feat: add config helper * feat: config helper * Update AuthServiceProvider.php --------- Co-authored-by: Rene Co-authored-by: René * fix: cms editor problem * feat: show forms in UI * fix: array * feat: add form locales (#287) * feat: locale support * chore: add todo * feat: send (sample) autolayout fields * feat: better names * fix: categories * feat: add Slider AutoLayout field * feat: update formbuilder option creation * feat: update layout package * feat: update layout package * fix: fixed form fields with repeatable options and added migration that checks the form tables for missing columns * fix: use correct object to get option name * fix: linting error * fix: remove aliases * fix: remove site specific packages * fix: remove .env file * fix: re-add packages * fix: remove default middleware * fix: update namespace * style: formatting --------- Co-authored-by: Xander Schuurman <44030544+keeama13@users.noreply.github.com> Co-authored-by: René Co-authored-by: Rene Co-authored-by: Merijn van Ginkel <107470233+nfmerijn@users.noreply.github.com> Co-authored-by: Jasper Co-authored-by: Merijn van Ginkel * fix: get string from nested option (#27) * feat: add requirement * fix: remove test credentials * style: formatting --------- Co-authored-by: Rene Co-authored-by: Xander Schuurman <44030544+keeama13@users.noreply.github.com> Co-authored-by: keeama13 Co-authored-by: thessakockelkorn <70509512+thessakockelkorn@users.noreply.github.com> Co-authored-by: Thessa Kockelkorn Co-authored-by: M.A. Peene Co-authored-by: Merijn van Ginkel <107470233+nfmerijn@users.noreply.github.com> Co-authored-by: Jasper Co-authored-by: Merijn van Ginkel * style: formatting * feat: Version 0.3.1 (#33) * fix!: field component (#30) * Fix: combination render (#31) * fix: render form fields with uppercase in type * fix: use field->GetCombinationChildren to (default) render combination fields * fix: lint * fix: fixed created_at and updated_at columns not setting values (#32) * feat!: load siteboss routes from normal route file --------- Co-authored-by: Xander Schuurman <44030544+keeama13@users.noreply.github.com> Co-authored-by: M.A. Peene Co-authored-by: Merijn van Ginkel <107470233+nfmerijn@users.noreply.github.com> Co-authored-by: Rene * Update issue templates --------- Co-authored-by: Rene Co-authored-by: Xander Schuurman <44030544+keeama13@users.noreply.github.com> Co-authored-by: keeama13 Co-authored-by: thessakockelkorn <70509512+thessakockelkorn@users.noreply.github.com> Co-authored-by: Thessa Kockelkorn Co-authored-by: M.A. Peene Co-authored-by: Merijn van Ginkel <107470233+nfmerijn@users.noreply.github.com> Co-authored-by: Jasper Co-authored-by: Merijn van Ginkel Co-authored-by: M.A. Peene --- .github/ISSUE_TEMPLATE/bug_report.md | 38 ++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..dd84ea78 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. iOS] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] + +**Smartphone (please complete the following information):** + - Device: [e.g. iPhone6] + - OS: [e.g. iOS8.1] + - Browser [e.g. stock browser, safari] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. From 2e44fe6ad7ca67436151aadd91e8753b27c0a4d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9?= Date: Mon, 3 Jul 2023 12:46:22 +0200 Subject: [PATCH 09/13] Update api.php --- routes/api.php | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/api.php b/routes/api.php index fb3745b6..0f6e4d93 100644 --- a/routes/api.php +++ b/routes/api.php @@ -10,6 +10,7 @@ use NotFound\Framework\Http\Controllers\SettingsController; use NotFound\Framework\Http\Controllers\Support\SupportController; use NotFound\Framework\Http\Controllers\UserPreferencesController; +use Siteboss\Routes\SiteRoutes; use Spatie\Honeypot\ProtectAgainstSpam; // ContentBlock From 7834c47d3b27c123af1435244fcd8a1a22ad79d6 Mon Sep 17 00:00:00 2001 From: thessakockelkorn <70509512+thessakockelkorn@users.noreply.github.com> Date: Tue, 4 Jul 2023 12:08:03 +0200 Subject: [PATCH 10/13] feat: multilingual search (#42) * fix!: field component (#30) * Fix: combination render (#31) * fix: render form fields with uppercase in type * fix: use field->GetCombinationChildren to (default) render combination fields * fix: lint * fix: fixed created_at and updated_at columns not setting values (#32) * feat!: load siteboss routes from normal route file * feat: language as string, language-specific fields * fix: right query for spellcheck * fix: variable names * fix: passing of language url * fix: pass page id for custom search values * feat: optional sorting of search results * fix: remove debug * fix: change language column into string * feat: use config for sitemap * style: formatting * fix: writing sitemap * fix: replace html tags by space * style: formatting * feat: error when solr hostname cannot be resolved * fix: remove unnecessary code * fix: pass localized url --------- Co-authored-by: Xander Schuurman <44030544+keeama13@users.noreply.github.com> Co-authored-by: M.A. Peene Co-authored-by: Merijn van Ginkel <107470233+nfmerijn@users.noreply.github.com> Co-authored-by: Rene Co-authored-by: Thessa Kockelkorn --- ...49_change_search_table_language_column.php | 27 ++++ .../mail/indexer/file-index-error.blade.php | 2 +- routes/api.php | 2 +- src/Models/Indexes/SolrIndex.php | 71 ++++++++--- src/Models/Indexes/SolrItem.php | 11 +- src/Services/Indexer/AbstractIndexService.php | 4 +- src/Services/Indexer/IndexBuilderService.php | 118 +++++++++--------- src/Services/Indexer/SolrIndexService.php | 5 +- 8 files changed, 156 insertions(+), 84 deletions(-) create mode 100644 database/migrations/2023_06_30_134549_change_search_table_language_column.php diff --git a/database/migrations/2023_06_30_134549_change_search_table_language_column.php b/database/migrations/2023_06_30_134549_change_search_table_language_column.php new file mode 100644 index 00000000..dee3acb2 --- /dev/null +++ b/database/migrations/2023_06_30_134549_change_search_table_language_column.php @@ -0,0 +1,27 @@ +string('language', 64)->change(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + // + } +}; diff --git a/resources/views/mail/indexer/file-index-error.blade.php b/resources/views/mail/indexer/file-index-error.blade.php index 57beb5ca..f2996563 100644 --- a/resources/views/mail/indexer/file-index-error.blade.php +++ b/resources/views/mail/indexer/file-index-error.blade.php @@ -1,5 +1,5 @@ @component('mail::message') -

Document {{ document }} op server {{ server }} geeft de volgende fout: {{ error }}

+

Document {{ $document }} op server {{ $server }} geeft de volgende fout: {{ $error }}

@endcomponent \ No newline at end of file diff --git a/routes/api.php b/routes/api.php index 0f6e4d93..6c380f83 100644 --- a/routes/api.php +++ b/routes/api.php @@ -10,7 +10,6 @@ use NotFound\Framework\Http\Controllers\SettingsController; use NotFound\Framework\Http\Controllers\Support\SupportController; use NotFound\Framework\Http\Controllers\UserPreferencesController; -use Siteboss\Routes\SiteRoutes; use Spatie\Honeypot\ProtectAgainstSpam; // ContentBlock @@ -24,6 +23,7 @@ | is assigned the "api" middleware group. Enjoy building your API! | */ + Route::prefix(config('siteboss.api_prefix'))->group(function () { // Unauthenticated routes Route::prefix('api')->group(function () { diff --git a/src/Models/Indexes/SolrIndex.php b/src/Models/Indexes/SolrIndex.php index a795d8d4..facd3733 100644 --- a/src/Models/Indexes/SolrIndex.php +++ b/src/Models/Indexes/SolrIndex.php @@ -130,13 +130,12 @@ public function testSolrConnection() return false; } - public function addOrUpdateItem(string $url, string $title, string $contents, string $type, int $lang, int $siteId, array $customValues, int $priority): bool + public function addOrUpdateItem(string $url, string $title, string $contents, string $type, string $lang, int $siteId, array $customValues, int $priority): bool { $curl = $this->solrHandler(); - $doc = [ - 'title' => $title, - 'content' => html_entity_decode(trim(preg_replace('/\s+/', ' ', strip_tags($contents)))), + sprintf('title_%s', $lang) => $title, + sprintf('content_%s', $lang) => html_entity_decode(trim(preg_replace('/\s+/', ' ', preg_replace('#<[^>]+>#', ' ', $contents)))), 'type' => $type, 'url' => $url, 'priority' => $priority, @@ -157,6 +156,11 @@ public function addOrUpdateItem(string $url, string $title, string $contents, st curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($payload)); $result = curl_exec($curl); + + if (curl_errno($curl) === 6) { + exit('[ERROR] Could not resolve solr host: '.$this->getSolrBaseUrl()); + } + $json = json_decode($result); if ($json && isset($json->responseHeader) && $json->responseHeader->status == 0) { return true; @@ -192,7 +196,7 @@ public function removeItem($url) return false; } - public function addOrUpdateFile(string $url, string $title, string $file, string $type, int $lang, int $siteId, array $customValues, int $priority): string + public function addOrUpdateFile(string $url, string $title, string $file, string $type, string $lang, int $siteId, array $customValues, int $priority): string { // find out of document exists $result = 0; @@ -202,9 +206,10 @@ public function addOrUpdateFile(string $url, string $title, string $file, string $curl = $this->solrHandler(); $endpoint = sprintf( - '%s/update/extract?literal.url=%s&literal.title=%s&literal.type=%s&literal.site=%s&literal.language=%d&commit=true', + '%s/update/extract?literal.url=%s&literal.title_%s=%s&literal.type=%s&literal.site=%s&literal.language=%d&commit=true', $this->getSolrBaseUrl(), urlencode($url), + $lang, urlencode($title), $type, $siteId, @@ -268,22 +273,25 @@ private function mailQueryError($query, $result) } } - public function selectItems($query, $filter = null, $start = null, $rows = null, $extraColumns = [], $highlightLength = 50) + public function selectItems($query, $lang = 'nl', $filter = null, $start = null, $rows = null, $extraColumns = [], $highlightLength = 50, $sortField = null, $sortDirection = 'desc') { $curl = $this->solrHandler(); $url = sprintf( - '%s/select?q=title:%s%%20content:%s&spellcheck.q=%s&wt=%s&hl=%s&q.op=%s&hl.fl=%s&fl=%s&spellcheck=true&hl.fragsize=%d&hl.maxAnalyzedChars=%d', + '%s/select?q=title_%s:%s%%20content_%s:%s&spellcheck.q=%s&wt=%s&hl=%s&q.op=%s&hl.fl=%s&fl=%s&spellcheck=true&hl.fragsize=%d&hl.maxAnalyzedChars=%d&spellcheck.dictionary=spellcheck_%s', $this->getSolrBaseUrl(), + $lang, + rawurlencode($query), // make sure + between search terms is preserved + $lang, rawurlencode($query), // make sure + between search terms is preserved rawurlencode($query), // make sure + between search terms is preserved - rawurlencode($query), // make sure + between search terms is preserved rawurlencode($query), // make sure + between search terms is preserved $this->wt, $this->hl, $this->selectOperator, - $this->hlfl, + sprintf('%s_%s', $this->hlfl, $lang), urlencode($this->fl), $this->hlfragsize, $this->hlmaxAnalyzedChars, + $lang ); if ($filter) { $url .= '&fq='.$filter; @@ -295,12 +303,11 @@ public function selectItems($query, $filter = null, $start = null, $rows = null, if ($rows && is_int($rows)) { $url .= '&rows='.$rows; } - if (count($extraColumns) > 0) { } - if ($this->sort) { - $url .= '&sort='.urlencode($this->sort); + if ($sortField) { + $url .= '&sort='.urlencode($sortField.' '.$sortDirection); } curl_setopt($curl, CURLOPT_URL, $url); @@ -359,7 +366,6 @@ public function buildSuggester() $url = sprintf('%s&suggest.build=true', $this->suggestUrl()); curl_setopt($curl, CURLOPT_URL, $url); - $result = curl_exec($curl); $json = json_decode($result); $searchResults = new SolrItem($json, null); @@ -367,10 +373,45 @@ public function buildSuggester() return $searchResults; } + private function getConfig() + { + $curl = $this->solrHandler(); + $url = sprintf('%s/config/searchComponent?componentName=suggest', $this->getSolrBaseUrl()); + curl_setopt($curl, CURLOPT_URL, $url); + curl_setopt($curl, CURLOPT_POST, false); + + $result = curl_exec($curl); + $json = json_decode($result); + + return $json; + } + + private function allSuggesters() + { + $json = $this->getConfig(); + $suggesters = []; + if ( + $json && isset($json->responseHeader) + && $json->responseHeader->status == 0 + && isset($json->config->searchComponent->suggest->suggester) + ) { + $list = $json->config->searchComponent->suggest->suggester; + foreach ($list as $s) { + $suggesters[] = $s->name; + } + } + + return $suggesters; + } + private function explodeSuggesters(): string { $suggesterString = ''; - foreach ($this->suggester as $s) { + $suggesters = $this->suggester; + if (count($suggesters) == 0) { + $suggesters = $this->allSuggesters(); + } + foreach ($suggesters as $s) { $suggesterString .= sprintf('&suggest.dictionary=%s', $s); } diff --git a/src/Models/Indexes/SolrItem.php b/src/Models/Indexes/SolrItem.php index ad627228..7f14e768 100644 --- a/src/Models/Indexes/SolrItem.php +++ b/src/Models/Indexes/SolrItem.php @@ -41,7 +41,7 @@ public function __construct($solr, $q, $collate = false, private int $highlightL $this->fl = explode(' ', $fl); $this->results = isset($solr->response->docs) ? $solr->response->docs : null; $this->highlights = isset($solr->highlighting) ? $solr->highlighting : null; - $this->spellcheck = $solr->spellcheck ?? null; + $this->spellcheck = isset($solr->spellcheck) ? $solr->spellcheck : null; $this->suggest = isset($solr->suggest) ? $solr->suggest : null; $this->number = isset($solr->response->numFound) ? $solr->response->numFound : 0; @@ -84,7 +84,8 @@ public function resultList() if ($column == 'url') { $resultArray[$column] = $this->parseUrl($result->{$column}); } else { - $resultArray[$column] = isset($result->{$column}[0]) ? $result->{$column}[0] : ''; + $columnName = preg_replace('/(_[a-zA-Z]{2}$)/', '', $column); + $resultArray[$columnName] = isset($result->{$column}) ? $result->{$column} : ''; } } @@ -186,12 +187,12 @@ public function spellcheckList() foreach ($this->spellcheck->suggestions as $suggestion) { if (isset($suggestion->startOffset)) { $suggest = substr($query, 0, $suggestion->startOffset).''.$suggestion->suggestion[0].''.substr($query, $suggestion->endOffset); - $suggest = preg_replace('/^([a-zA-Z])+:/', '', $suggest); // remove search field if necessary + $suggestTerm = preg_replace('/^([a-zA-Z])+(_[a-zA-Z]{2})?:/', '', $suggest); // remove search field if necessary $suggest_url = substr($query, 0, $suggestion->startOffset).$suggestion->suggestion[0].substr($query, $suggestion->endOffset); - $suggest_url = preg_replace('/^([a-zA-Z])+:/', '', $suggest_url); // remove search field if necessary + $suggest_url = preg_replace('/^([a-zA-Z])+(_[a-zA-Z]{2})?:/', '', $suggest_url); // remove search field if necessary - $items[] = (object) ['link' => '?q='.rawurlencode(urldecode($suggest_url)), 'text' => urldecode($suggest)]; + $items[] = (object) ['link' => '?q='.rawurlencode(urldecode($suggest_url)), 'text' => urldecode($suggestTerm)]; } } } diff --git a/src/Services/Indexer/AbstractIndexService.php b/src/Services/Indexer/AbstractIndexService.php index c142af30..0add32a8 100644 --- a/src/Services/Indexer/AbstractIndexService.php +++ b/src/Services/Indexer/AbstractIndexService.php @@ -18,7 +18,7 @@ abstract public function finishUpdate(): object; abstract public function urlNeedsUpdate(string $url, $updated): bool; - abstract public function upsertUrl(string $url, string $title, string $contents, string $type, int $lang, array $customValues = []): object; + abstract public function upsertUrl(string $url, string $title, string $contents, string $type, string $lang, array $customValues = []): object; - abstract public function upsertFile(string $url, string $title, string $file, string $type, int $lang, array $customValues): object; + abstract public function upsertFile(string $url, string $title, string $file, string $type, string $lang, array $customValues): object; } diff --git a/src/Services/Indexer/IndexBuilderService.php b/src/Services/Indexer/IndexBuilderService.php index c77fa385..5e70796f 100644 --- a/src/Services/Indexer/IndexBuilderService.php +++ b/src/Services/Indexer/IndexBuilderService.php @@ -13,9 +13,7 @@ class IndexBuilderService private $locales; - private $defaultLocale; - - private $domainName; + private $domain; private $sitemapFile; @@ -26,9 +24,7 @@ public function __construct(string $serverType, $debug = false) $this->debug = $debug; $this->locales = Lang::all(); - $locale = env('SB_LOCALES_DEFAULT', 'nl'); - $this->defaultLocale = Lang::where('url', $locale)->get(); - $this->domainName = env('APP_NAME'); + $this->domain = rtrim(env('APP_URL', ''), '/'); switch ($serverType) { case 'solr': $this->searchServer = new SolrIndexService($this->debug); @@ -50,7 +46,7 @@ public function run() foreach ($sites as $site) { $siteName = $site->name; - $sitemapFileName = env('APP_SITEMAP'); + $sitemapFileName = config('solr.sitemap'); if ($sitemapFileName) { $this->createFolderIfNotExists($sitemapFileName); $this->sitemapFile = fopen($sitemapFileName, 'w') or exit('Could not open sitemap file for writing'); @@ -104,15 +100,10 @@ private function indexChildPages($parentId) $menu = Menu::whereId($page->id)->firstOrFail(); - if ($this->searchServer->urlNeedsUpdate($menu->getPath(), strtotime($menu->updated_at))) { - $this->writeDebug(': update needed: '); - - foreach ($this->locales as $lang) { - $this->updatePage($menu, $lang); - } - } else { - $this->writeDebug(": Does not need updating\n"); + foreach ($this->locales as $lang) { + $this->updatePage($menu, $lang); } + // index subitems for page foreach ($this->locales as $lang) { $this->updateSubPages($menu, $lang); @@ -124,59 +115,63 @@ private function indexChildPages($parentId) private function updatePage($menu, $lang) { - $success = true; app()->setLocale($lang->url); - - if ($this->sitemapFile) { - $sitemap = ''; - } - $searchText = ''; - $pageService = new PageService($menu, $lang); - $title = $menu->getTitle($lang); if (count($this->locales) == 1) { $url = $menu->getPath(); } else { $url = $menu->getLocalizedPath(); } - $searchText = $pageService->getContentForIndexer(); - // continue with customValues - $customValues = []; + if ($this->searchServer->urlNeedsUpdate($url, strtotime($menu->updated_at))) { + $this->writeDebug(': update needed: '); - $class = $menu->template->filename ?? ''; - $className = 'App\Http\Controllers\Page\\'.$class.'Controller'; - $c = null; - $priority = 1; - if (class_exists($className)) { - $c = new $className(); - if (method_exists($className, 'customSearchValues')) { - $customValues = $c->customSearchValues(); - } - if (method_exists($className, 'searchPriority')) { - $priority = $c->searchPriority(); - } - } + $searchText = ''; + $pageService = new PageService($menu, $lang); + $title = $menu->getTitle($lang); - $searchText = rtrim($searchText, ', '); - if (! empty($title) && ! empty($searchText)) { - $result = $this->searchServer->upsertUrl($url, $title, $searchText, 'page', $lang->id, $customValues, $priority); + $searchText = $pageService->getContentForIndexer(); - if ($result->errorCode == 0) { - $this->writeDebug(" success\n"); - } else { - $this->writeDebug(" FAILED\n"); + // continue with customValues + $customValues = []; + + $class = $menu->template->filename ?? ''; + $className = 'App\Http\Controllers\Page\\'.$class.'Controller'; + $c = null; + $priority = 1; + if (class_exists($className)) { + $c = new $className(); + if (method_exists($className, 'customSearchValues')) { + $customValues = $c->customSearchValues($menu->id); + } + if (method_exists($className, 'searchPriority')) { + $priority = $c->searchPriority(); + } } - if ($this->sitemapFile) { - // update sitemap - $sitemap .= sprintf( - "%s%s\r\n", - $this->domainName, - $url - ); + $searchText = rtrim($searchText, ', '); + if (! empty($title) && ! empty($searchText)) { + $result = $this->searchServer->upsertUrl($url, $title, $searchText, 'page', $lang->url, $customValues, $priority); + + if ($result->errorCode == 0) { + $this->writeDebug(" success\n"); + } else { + $this->writeDebug(" FAILED\n"); + } + } else { + $this->writeDebug(" empty page or title\n"); } } else { - $this->writeDebug(" empty page or title\n"); + $this->writeDebug(": Does not need updating\n"); + } + + if ($this->sitemapFile) { + // update sitemap + $sitemap = sprintf( + "%s%s\r\n", + $this->domain, + $url + ); + fwrite($this->sitemapFile, $sitemap); } } @@ -207,15 +202,15 @@ private function updateSubitems($class, $lang) $success = true; if ($searchItem['isFile']) { - $success = $this->searchServer->upsertFile($url, $searchItem['title'], $searchItem['file'], $searchItem['type'], $lang->id, $searchItem['customValues'], $searchItem['priority']); + $success = $this->searchServer->upsertFile($url, $searchItem['title'], $searchItem['file'], $searchItem['type'], $lang->url, $searchItem['customValues'], $searchItem['priority']); } else { // subitem is table row - $success = $this->searchServer->upsertUrl($url, $searchItem['title'], $searchItem['content'], $searchItem['type'], $lang->id, $searchItem['customValues'], $searchItem['priority']); + $success = $this->searchServer->upsertUrl($url, $searchItem['title'], $searchItem['content'], $searchItem['type'], $lang->url, $searchItem['customValues'], $searchItem['priority']); } if ($this->sitemapFile && $searchItem['sitemap']) { $sitemap = sprintf( "%s%s\r\n", - $this->domainName, + $this->domain, $url ); } @@ -228,6 +223,15 @@ private function updateSubitems($class, $lang) } else { $this->writeDebug(": Does not need updating\n"); } + + if ($this->sitemapFile) { + $sitemap = sprintf( + "%s%s\r\n", + $this->domain, + $url + ); + fwrite($this->sitemapFile, $sitemap); + } } } } diff --git a/src/Services/Indexer/SolrIndexService.php b/src/Services/Indexer/SolrIndexService.php index 92aa1dc9..22c59b8b 100644 --- a/src/Services/Indexer/SolrIndexService.php +++ b/src/Services/Indexer/SolrIndexService.php @@ -32,7 +32,7 @@ public function urlNeedsUpdate(string $url, $updated): bool return true; } - public function upsertUrl(string $url, string $title, string $contents, string $type, int $lang, array $customValues = [], $priority = 1): object + public function upsertUrl(string $url, string $title, string $contents, string $type, string $lang, array $customValues = [], $priority = 1): object { $result = $this->solrIndex->addOrUpdateItem($this->siteUrl($url), $title, $contents, $type, $lang, $this->siteId, $customValues, $priority); $return = $this->returnvalue(); @@ -51,7 +51,7 @@ public function upsertUrl(string $url, string $title, string $contents, string $ return $return; } - public function upsertFile(string $url, string $title, string $file, string $type, int $lang, array $customValues = [], $priority = 1): object + public function upsertFile(string $url, string $title, string $file, string $type, string $lang, array $customValues = [], $priority = 1): object { $result = $this->solrIndex->addOrUpdateFile($this->siteUrl($url), $title, $file, $type, $lang, $this->siteId, $customValues, $priority); @@ -107,7 +107,6 @@ public function finishUpdate(): object { $return = $this->removeAllPending(); - $this->solrIndex->suggester = ['fulltextsuggester', 'titlesuggester']; $build = $this->solrIndex->buildSuggester(); if ($build->error) { From ddd3b9f87d38ff0b7a506ca228183e51b70f5ce9 Mon Sep 17 00:00:00 2001 From: Rene Date: Tue, 4 Jul 2023 12:47:45 +0200 Subject: [PATCH 11/13] fix: correct license in composer file --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 041df2fa..2ad8b43c 100644 --- a/composer.json +++ b/composer.json @@ -7,7 +7,7 @@ "siteboss-framework" ], "homepage": "https://github.com/notfoundnl/siteboss-framework", - "license": "MIT", + "license": "AGPL-3.0-or-later", "authors": [ { "name": "NotFound Digital Creativity", From e76dd7be9b7fb1d45bccadca893c37a5b137db67 Mon Sep 17 00:00:00 2001 From: Rene Date: Tue, 4 Jul 2023 14:56:00 +0200 Subject: [PATCH 12/13] feat: add API routes --- src/Providers/RouteServiceProvider.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Providers/RouteServiceProvider.php b/src/Providers/RouteServiceProvider.php index 2a95039a..6cb79061 100644 --- a/src/Providers/RouteServiceProvider.php +++ b/src/Providers/RouteServiceProvider.php @@ -42,6 +42,15 @@ public function boot() Route::middleware('web') ->group(base_path('routes/web.php')); + + + // Optionally load API routes + if( file_exists(base_path('routes/api.php'))) + { + Route::middleware('api') + ->prefix('api') + ->group( base_path('routes/api.php')); + } }); } @@ -54,4 +63,4 @@ protected function configureRateLimiting() return Limit::perMinute(60)->by(optional($request->user())->id ?: $request->ip()); }); } -} +} \ No newline at end of file From 08abb6a79c275b969589b47bb667b55ff9fde1da Mon Sep 17 00:00:00 2001 From: Rene Date: Tue, 4 Jul 2023 15:02:28 +0200 Subject: [PATCH 13/13] style: formatting --- src/Providers/RouteServiceProvider.php | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Providers/RouteServiceProvider.php b/src/Providers/RouteServiceProvider.php index 6cb79061..bc935c0d 100644 --- a/src/Providers/RouteServiceProvider.php +++ b/src/Providers/RouteServiceProvider.php @@ -43,13 +43,11 @@ public function boot() Route::middleware('web') ->group(base_path('routes/web.php')); - // Optionally load API routes - if( file_exists(base_path('routes/api.php'))) - { + if (file_exists(base_path('routes/api.php'))) { Route::middleware('api') ->prefix('api') - ->group( base_path('routes/api.php')); + ->group(base_path('routes/api.php')); } }); } @@ -63,4 +61,4 @@ protected function configureRateLimiting() return Limit::perMinute(60)->by(optional($request->user())->id ?: $request->ip()); }); } -} \ No newline at end of file +}