Skip to content

Commit

Permalink
Merge branch 'develop' into feat/optimize-using-cache
Browse files Browse the repository at this point in the history
  • Loading branch information
@64knl
64knl authored Apr 3, 2024
2 parents 8cedde9 + 3ace122 commit 900226f
Show file tree
Hide file tree
Showing 6 changed files with 35 additions and 22 deletions.
3 changes: 1 addition & 2 deletions composer.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,11 @@
"spatie/laravel-honeypot": "^4.3.2",
"illuminate/contracts": "^10.0|^11.0",
"notfoundnl/siteboss-layout": "^1.6.1",
"notfoundnl/siteboss-static": "^1.13",
"notfoundnl/siteboss-static": "^1.15.0",
"mcamara/laravel-localization": "^2.0",
"xenolope/quahog": "^3.0",
"firebase/php-jwt": "^6.3",
"intervention/image": "^3.0",
"php": "^8.1",
"doctrine/dbal": "^3.6"
},
"require-dev": {
Expand Down
6 changes: 3 additions & 3 deletions routes/console.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@
use NotFound\Framework\Services\CmsExchange\ExchangeConsoleService;
use NotFound\Framework\Services\Indexer\IndexBuilderService;

Artisan::command('siteboss:index-site {--debug : Whether debug messages should be displayed} {--clean : Truncate search table}', function ($debug, $clean) {
Artisan::command('siteboss:index-site {--debug : Display debug messages} {--fresh : Empty local search table}', function ($debug, $fresh) {

$indexer = new IndexBuilderService($debug, $clean);
$indexer = new IndexBuilderService($debug, $fresh);
$indexer->run();

return Command::SUCCESS;
})->purpose('Index site for local search');

Artisan::command('siteboss:cms-import {--debug : Whether debug messages should be displayed} {--dry : Dry Run}', function ($debug, $dry) {
Artisan::command('siteboss:cms-import {--debug : Display debug messages} {--dry : Dry Run}', function ($debug, $dry) {

$indexer = new ExchangeConsoleService($debug, $dry);
$indexer->import();
Expand Down
16 changes: 10 additions & 6 deletions src/Models/Indexes/SolrIndex.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,14 +131,14 @@ public function testSolrConnection()
return false;
}

public function upsertItem(SearchItem $indexItem, int $siteId = 1): bool
public function upsertItem(SearchItem $indexItem, int $siteId = 1, ?string $domain = null): bool
{
$curl = $this->solrHandler();
$doc = [
sprintf('title_%s', $indexItem->language()) => $indexItem->title(),
sprintf('content_%s', $indexItem->language()) => html_entity_decode(trim(preg_replace('/\s+/', ' ', preg_replace('#<[^>]+>#', ' ', $indexItem->content())))),
'type' => $indexItem->type(),
'url' => $this->siteUrl($indexItem->url(), $siteId),
'url' => $this->siteUrl($indexItem->url(), $domain),
'priority' => $indexItem->priority(),
'site' => $siteId,
'language' => $indexItem->language(),
Expand Down Expand Up @@ -197,7 +197,7 @@ public function removeItem($url)
return false;
}

public function upsertFile(SearchItem $indexItem, int $siteId = 1): string
public function upsertFile(SearchItem $indexItem, int $siteId = 1, ?string $domain = null): string
{

// find out of document exists
Expand All @@ -209,7 +209,7 @@ public function upsertFile(SearchItem $indexItem, int $siteId = 1): string
$endpoint = sprintf(
'%s/update/extract?literal.url=%s&literal.title_%s=%s&literal.type=%s&literal.site=%s&literal.language=%d&literal.solr_date=%s&uprefix=ignored_&fmap.content=content_%s&commit=true',
$this->getSolrBaseUrl(),
urlencode($this->siteUrl($indexItem->url(), $siteId)),
urlencode($this->siteUrl($indexItem->url(), $domain)),
$indexItem->language(),
urlencode($indexItem->title()),
$indexItem->type(),
Expand Down Expand Up @@ -441,8 +441,12 @@ public function checkConnection(): bool
return false;
}

public function siteUrl($url, $siteId): string
public function siteUrl($url, ?string $domain): string
{
return sprintf('{{%d}}%s', $siteId, $url);
if ($domain) {
return sprintf('%s/%s', rtrim($domain, '/'), ltrim($url, '/'));
}

return $url;
}
}
10 changes: 8 additions & 2 deletions src/Services/Indexer/AbstractIndexService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ abstract class AbstractIndexService

public int $siteId;

public ?string $domain;

abstract public function __construct($debug = false);

abstract public function startUpdate(): bool;
Expand All @@ -29,7 +31,7 @@ public function clean(): bool

public function urlNeedsUpdate(string $url, $updated): bool
{
$searchItem = CmsSearch::whereUrl($url)->first();
$searchItem = CmsSearch::whereUrl($this->siteUrl($url))->first();
if ($searchItem && ($searchItem->updated_at !== null && $searchItem->updated_at->timestamp > $updated)) {
CmsSearch::whereUrl($url)->update(['search_status' => 'SKIPPED']);

Expand All @@ -41,6 +43,10 @@ public function urlNeedsUpdate(string $url, $updated): bool

protected function siteUrl($url): string
{
return sprintf('{{%d}}%s', $this->siteId, $url);
if ($this->domain) {
return sprintf('%s/%s', rtrim($this->domain, '/'), ltrim($url, '/'));
}

return $url;
}
}
10 changes: 6 additions & 4 deletions src/Services/Indexer/IndexBuilderService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ class IndexBuilderService
{
private bool $debug;

private bool $clean;
private bool $fresh;

private $locales;

Expand All @@ -22,11 +22,11 @@ class IndexBuilderService

private AbstractIndexService $searchServer;

public function __construct($debug = false, $clean = false)
public function __construct($debug = false, $fresh = false)
{
$serverType = config('indexer.engine');
$this->debug = $debug;
$this->clean = $clean;
$this->fresh = $fresh;
$this->locales = Lang::all();

$this->domain = rtrim(env('APP_URL', ''), '/');
Expand All @@ -46,7 +46,7 @@ public function run()

return;
}
if ($this->clean) {
if ($this->fresh) {
$this->searchServer->clean();
}
$sites = CmsSite::whereIndex(1)->get();
Expand All @@ -69,6 +69,8 @@ public function run()

$siteId = $site->id;
$this->searchServer->siteId = $siteId;
$this->searchServer->domain = $site->domain ?? null;

$this->searchServer->languageId = 1;

// insert all pages, starting from the root
Expand Down
12 changes: 7 additions & 5 deletions src/Services/Indexer/SolrIndexService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ class SolrIndexService extends AbstractIndexService

public int $siteId;

public ?string $domain;

public $solrIndex;

public function __construct($debug = false)
Expand All @@ -26,7 +28,7 @@ public function upsertItem(SearchItem $searchItem): object
$cmsSearchItemStatus = '';

if ($searchItem->type() === 'file') {
$result = $this->solrIndex->upsertFile($searchItem, $this->siteId);
$result = $this->solrIndex->upsertFile($searchItem, $this->siteId, $this->domain);

$return = $this->returnvalue();
if ($result == 'success') {
Expand All @@ -37,7 +39,7 @@ public function upsertItem(SearchItem $searchItem): object
$return->message = "failed: file not found \n";
} else {
$cmsSearchItemStatus = 'NOT_INDEXABLE';
$result = $this->solrIndex->upsertItem($searchItem, $this->siteId);
$result = $this->solrIndex->upsertItem($searchItem, $this->siteId, $this->domain);
if ($result) {
$cmsSearchItemStatus = 'UPDATED';
} else {
Expand All @@ -46,7 +48,7 @@ public function upsertItem(SearchItem $searchItem): object
}
}
} else {
$result = $this->solrIndex->upsertItem($searchItem, $this->siteId);
$result = $this->solrIndex->upsertItem($searchItem, $this->siteId, $this->domain);

if ($result) {
$cmsSearchItemStatus = 'UPDATED';
Expand All @@ -57,9 +59,9 @@ public function upsertItem(SearchItem $searchItem): object
}
}

$cmsSearchItem = CmsSearch::firstOrNew(['url' => $this->solrIndex->siteUrl($searchItem->url(), $this->siteId)]);
$cmsSearchItem = CmsSearch::firstOrNew(['url' => $this->solrIndex->siteUrl($searchItem->url(), $this->domain)]);
$cmsSearchItem->setValues($searchItem, $cmsSearchItemStatus);
$cmsSearchItem->url = $this->solrIndex->siteUrl($searchItem->url(), $this->siteId);
$cmsSearchItem->url = $this->solrIndex->siteUrl($searchItem->url(), $this->domain);
$cmsSearchItem->save();
if ($cmsSearchItemStatus == 'FAILED') {
$cmsSearchItem->updated_at = null;
Expand Down

0 comments on commit 900226f

Please sign in to comment.