Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat!:Version 0.21.0 #234

Merged
merged 10 commits into from
May 24, 2024
Merged

feat!:Version 0.21.0 #234

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
cb87c4f
fix: solr accents (#232)
64knl Apr 18, 2024
ef47c5b
fix(search): retain skipped items
64knl May 8, 2024
3564c15
fix(solr): re-index failed document
64knl May 8, 2024
86f54df
style: formatting
64knl May 15, 2024
b0416f5
feat: add function to abstract class.
64knl May 15, 2024
ecb7ef9
chore: remove SolrDate
64knl May 15, 2024
64ec8d6
feat: empty core
64knl May 15, 2024
d5405b2
fix: syntax problem
64knl May 16, 2024
a63f6ac
feat: DateTime for SearchItem
64knl May 16, 2024
1dfaa7a
style: formatting
64knl May 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 0 additions & 11 deletions src/Http/Controllers/Pages/PageController.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,15 +171,4 @@ public function searchSubitems(): array
{
return [];
}

public function solrDate($pageId)
{
$this->pageId = $pageId;
$date = $this->menuItem()->updated_at;
if ($date) {
return $date->toIso8601String();
} else {
return '2023-01-01T10:00:00Z';
}
}
}
7 changes: 6 additions & 1 deletion src/Models/CmsSearch.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,12 @@ class CmsSearch extends BaseModel
{
protected $table = 'cms_search';

private array $status = ['PENDING', 'ADDED', 'SKIPPED', 'UPDATED', 'NOT_INDEXABLE', 'NOT_FOUND'];
protected $casts = [
'updated_at' => 'datetime',
'created_at' => 'datetime',
'deleted_at' => 'datetime',
];
// private array $status = ['PENDING', 'ADDED', 'SKIPPED', 'UPDATED', 'NOT_INDEXABLE', 'NOT_FOUND'];

private static array $skipStatus = ['NOT_INDEXABLE', 'NOT_FOUND'];

Expand Down
26 changes: 10 additions & 16 deletions src/Models/Indexes/SolrIndex.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
use NotFound\Framework\Mail\Indexer\FileIndexError;
use NotFound\Framework\Mail\Indexer\QueryError;
use NotFound\Framework\Models\BaseModel;
use NotFound\Framework\Models\CmsSearch;
use NotFound\Framework\Services\Indexer\SearchItem;

/**
Expand Down Expand Up @@ -71,25 +70,20 @@ public function __construct($debug = false)

public function emptyCore()
{
$searchItems = CmsSearch::all();
if (count($searchItems) == 0) {
$curl = $this->solrHandler();
$url = sprintf('%s/update/?wt=%s&commit=true*', $this->getSolrBaseUrl(), $this->wt);
curl_setopt($curl, CURLOPT_URL, $url);
$payload = ['delete' => ['query' => '*:*']];

curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($payload));
$result = curl_exec($curl);
$curl = $this->solrHandler();
$url = sprintf('%s/update/?wt=%s&commit=true*', $this->getSolrBaseUrl(), $this->wt);
curl_setopt($curl, CURLOPT_URL, $url);
$payload = ['delete' => ['query' => '*:*']];

$json = json_decode($result);
curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($payload));
$result = curl_exec($curl);

if (! $json || ! isset($json->responseHeader) || $json->responseHeader->status !== 0) {
$this->mailQueryError($url, $result);
$json = json_decode($result);

return false;
}
if (! $json || ! isset($json->responseHeader) || $json->responseHeader->status !== 0) {
$this->mailQueryError($url, $result);

return true;
return false;
}

return true;
Expand Down
13 changes: 10 additions & 3 deletions src/Services/Indexer/AbstractIndexService.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

namespace NotFound\Framework\Services\Indexer;

use DateTime;
use NotFound\Framework\Models\CmsSearch;

abstract class AbstractIndexService
Expand All @@ -12,14 +13,20 @@ abstract class AbstractIndexService

public ?string $domain;

abstract public function __construct($debug = false);
protected bool $debug;

protected bool $fresh;

abstract public function __construct(bool $debug = false, bool $fresh = false);

abstract public function startUpdate(): bool;

abstract public function finishUpdate(): object;

abstract public function upsertItem(SearchItem $searchItem): object;

abstract public function retainItem(string $url): void;

abstract public function checkConnection(): bool;

public function clean(): bool
Expand All @@ -29,10 +36,10 @@ public function clean(): bool
return true;
}

public function urlNeedsUpdate(string $url, $updated): bool
public function urlNeedsUpdate(string $url, ?DateTime $updated): bool
{
$searchItem = CmsSearch::whereUrl($this->siteUrl($url))->first();
if ($searchItem && ($searchItem->updated_at !== null && $searchItem->updated_at->timestamp > $updated)) {
if ($searchItem && ($searchItem->updated_at !== null && $searchItem->updated_at >= $updated)) {
CmsSearch::whereUrl($url)->update(['search_status' => 'SKIPPED']);

return false;
Expand Down
74 changes: 40 additions & 34 deletions src/Services/Indexer/IndexBuilderService.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,17 @@

class IndexBuilderService
{
private bool $debug;

private bool $fresh;

private $locales;

private $domain;

private $sitemapFile;

private $padding = 140;

private AbstractIndexService $searchServer;

public function __construct($debug = false, $fresh = false)
public function __construct(private bool $debug = false, private bool $fresh = false)
{
$serverType = config('indexer.engine');
$this->debug = $debug;
Expand All @@ -32,14 +30,14 @@ public function __construct($debug = false, $fresh = false)
$this->domain = rtrim(env('APP_URL', ''), '/');
switch ($serverType) {
case 'solr':
$this->searchServer = new SolrIndexService($this->debug);
$this->searchServer = new SolrIndexService($this->debug, $this->fresh);
break;
default:
exit('Unknown search index type');
}
}

public function run()
public function run(): void
{
if (! $this->searchServer->checkConnection()) {
$this->writeDebug("\n\n Error connecting to search server! \n\n");
Expand All @@ -64,7 +62,7 @@ public function run()
$this->sitemapFile = fopen($sitemapFileName, 'w') or exit('Could not open sitemap file for writing');
} else {
$this->sitemapFile = false;
$this->writeDebug(" skipping sitemap\n");
$this->writeDebug(" ⏭️ Skipping sitemap\n");
}

$siteId = $site->id;
Expand All @@ -74,41 +72,42 @@ public function run()
$this->searchServer->languageId = 1;

// insert all pages, starting from the root
$this->writeDebug(" INDEXING PAGES\n ==============\n");
$this->writeDebug("┃ INDEXING PAGES\n ==============\n");
$this->indexChildPages($site->root);

if ($this->sitemapFile) {
fclose($this->sitemapFile);
}
$finish = $this->searchServer->finishUpdate();

$this->writeDebug($finish->message);
}
} else {
$this->writeDebug("No sites to index\n");

return;
}
$finish = $this->searchServer->finishUpdate();

$this->writeDebug($finish->message);
}

private function indexChildPages($parentId)
{
$childPages = Menu::whereParent_id($parentId)->whereEnabled(1)->get();
foreach ($childPages as $page) {
$this->writeDebug(sprintf(" * Page \e[1m%s\e[0m (id: %d)", $page->url, $page->id));
$this->writeDebug("┃\n");
$this->writeDebug(sprintf('%s (id: %d)', $page->url, $page->id), true, '┣━┓ 📂 Page ');

if (! isset($page->template->id)) {
$this->writeDebug(" skipping, no template found\n");
$this->writeDebug(": ❌ Fail, skipping, no template found\n");

continue;
}

$menu = Menu::whereId($page->id)->firstOrFail();

if (! isset($page->template->properties->searchable) || $page->template->properties->searchable == 0) {
$this->writeDebug(" skipping, template not searchable\n");

$this->writeDebug(": ⏭️ Skipping, template excluded from search\n");
} elseif (isset($page->properties->excludeFromSearch) && $page->properties->excludeFromSearch == true) {
$this->writeDebug(" skipping, page not searchable\n");

$this->writeDebug(": ⏭️ Skipping, page excluded from search\n");
} else {

foreach ($this->locales as $lang) {
Expand All @@ -133,7 +132,7 @@ private function updatePage($menu, $lang)
$url = $menu->getLocalizedPath();
}

if ($this->searchServer->urlNeedsUpdate($url, strtotime($menu->updated_at))) {
if ($this->searchServer->urlNeedsUpdate($url, $menu->updated_at)) {
$this->writeDebug(': update needed: ');

$searchText = '';
Expand All @@ -149,7 +148,7 @@ private function updatePage($menu, $lang)

$c = null;
$priority = 1;
$solrDate = '';

if (class_exists($className)) {
$c = new $className();
if (method_exists($className, 'customSearchValues')) {
Expand All @@ -158,30 +157,31 @@ private function updatePage($menu, $lang)
if (method_exists($className, 'searchPriority')) {
$priority = $c->searchPriority();
}
if (method_exists($className, 'solrDate')) {
$solrDate = $c->solrDate($menu->id);
}
}
$searchText = rtrim($searchText, ', ');
if (! empty($title) && ! empty($searchText)) {

$searchItem = new SearchItem($url, $title);
$searchItem->setContent($searchText)->setLanguage($lang->url)->setPriority($priority)->setPublicationDate(new DateTime($menu->updated_at));
$searchItem->setContent($searchText)
->setLanguage($lang->url)
->setPriority($priority)
->setPublicationDate(new DateTime($menu->updated_at));
foreach ($customValues as $key => $value) {
$searchItem->setCustomValue($key, $value);
}
$result = $this->searchServer->upsertItem($searchItem);

if ($result->errorCode == 0) {
$this->writeDebug(" success\n");
$this->writeDebug(": ✅ Success\n");
} else {
$this->writeDebug(" FAILED\n");
$this->writeDebug(": ❌ FAILED\n");
}
} else {
$this->writeDebug(" empty page or title\n");
$this->writeDebug(": ❌ Empty page or title\n");
}
} else {
$this->writeDebug(": Does not need updating\n");
$this->writeDebug(": ✅ Page does not need updating\n");
$this->searchServer->retainItem($url);
}

if ($this->sitemapFile) {
Expand Down Expand Up @@ -235,9 +235,9 @@ private function updateSubitems($class, $lang)
$success = false;
if ((new \ReflectionClass($searchItem))->getShortName() == 'SearchItem') {
$url = $searchItem->url();
$this->writeDebug($url);
$this->writeDebug($url, true, '┃ ┣━ 📄 ');

if ($this->searchServer->urlNeedsUpdate($url, strtotime($searchItem->lastUpdated()))) {
if ($this->searchServer->urlNeedsUpdate($url, $searchItem->lastUpdated())) {

$searchItem->setLanguage($lang->url);
$success = $this->searchServer->upsertItem($searchItem);
Expand All @@ -251,12 +251,13 @@ private function updateSubitems($class, $lang)
}

if ($success->errorCode == 0) {
$this->writeDebug(" success\n");
$this->writeDebug(": ✅ Success\n");
} else {
$this->writeDebug($success->message);
}
} else {
$this->writeDebug(": Does not need updating\n");
$this->writeDebug(": ✅ Item does not need updating\n");
$success = $this->searchServer->retainItem($url);
}
} else {
dd('Please use the SearchItem class');
Expand All @@ -275,10 +276,15 @@ private function createFolderIfNotExists($fullFilePath)
}
}

private function writeDebug($text)
private function writeDebug($text, $padding = false, $prefix = '')
{
if ($this->debug) {
printf($text);

if ($padding) {
$text = substr($text, 0, $this->padding - strlen($prefix));
$text = str_pad($text, $this->padding - strlen($prefix), ' ');
}
printf("\e[1m".$prefix."\e[0m".$text);
}
}

Expand Down
5 changes: 3 additions & 2 deletions src/Services/Indexer/SearchItem.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

use DateTime;
use NotFound\Framework\Models\Lang;
use NotFound\Framework\View\Components\Forms\Fields\Date;

final class SearchItem
{
Expand Down Expand Up @@ -155,9 +156,9 @@ public function publicationDate(): ?string
* This is when the content was last updated
* This is used to determine if the content needs to be re-indexed
*/
public function lastUpdated(): ?string
public function lastUpdated(): ?DateTime
{
return $this->toDateString($this->lastUpdated ?? $this->publicationDate);
return $this->lastUpdated ?? $this->publicationDate;
}

public function customValues(): ?array
Expand Down
Loading
Loading