Post Browser Navigate Hooks

Adds post browser navigate hooks that are passed from `Http` steps to
the browser loader helper and are executed after the headless browser
navigated to the specified URL. They are called with the chrome-php
`Page` object as argument, so you can run things on that page before
getting the HTML and returning the response.

CHANGELOG.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.1.0] - 2024-10-17
+### Added
+* The new `postBrowserNavigateHook()` method in the `Http` step classes, which allows to define callback functions that are triggered after the headless browser navigated to the specified URL. They are called with the chrome-php `Page` object as argument.
+
 ## [2.0.1] - 2024-10-15
 ### Fixed
 * Issue with the `afterLoad` hook of the `HttpLoader`, introduced in v2. Calling the hook was commented out, which slipped through because the test case was faulty.

src/Loader/Http/HeadlessBrowserLoaderHelper.php

@@ -2,6 +2,7 @@
 
 namespace Crwlr\Crawler\Loader\Http;
 
+use Closure;
 use Crwlr\Crawler\Loader\Http\Messages\RespondedRequest;
 use Crwlr\Crawler\Loader\Http\Politeness\Throttler;
 use Exception;
@@ -42,8 +43,28 @@ class HeadlessBrowserLoaderHelper
 
     protected int $timeout = 30_000;
 
+    /**
+     * @var Closure[]
+     */
+    protected array $tempPostNavigateHooks = [];
+
     public function __construct(private ?BrowserFactory $browserFactory = null) {}
 
+    /**
+     * Set temporary post navigate hooks
+     *
+     * They will be executed after the next call to navigateToPageAndGetRespondedRequest()
+     * and forgotten afterward.
+     *
+     * @param Closure[] $hooks
+     */
+    public function setTempPostNavigateHooks(array $hooks): static
+    {
+        $this->tempPostNavigateHooks = $hooks;
+
+        return $this;
+    }
+
     /**
      * @throws OperationTimedOut
      * @throws CommunicationException
@@ -81,6 +102,8 @@ function ($params) use (&$statusCode, &$responseHeaders) {
 
         $this->navigate($request->getUri()->__toString());
 
+        $this->callPostNavigateHooks();
+
         $throttler->trackRequestEndFor($request->getUri());
 
         $html = $this->page?->getHtml();
@@ -196,6 +219,17 @@ protected function navigate(string $url): void
         }
     }
 
+    protected function callPostNavigateHooks(): void
+    {
+        if (!empty($this->tempPostNavigateHooks)) {
+            foreach ($this->tempPostNavigateHooks as $hook) {
+                $hook->call($this, $this->page);
+            }
+        }
+
+        $this->tempPostNavigateHooks = [];
+    }
+
     /**
      * @throws Exception
      */

src/Steps/Loading/HttpBase.php

@@ -2,6 +2,7 @@
 
 namespace Crwlr\Crawler\Steps\Loading;
 
+use Closure;
 use Crwlr\Crawler\Loader\Http\Exceptions\LoadingException;
 use Crwlr\Crawler\Loader\Http\HttpLoader;
 use Crwlr\Crawler\Loader\Http\Messages\RespondedRequest;
@@ -42,6 +43,11 @@ abstract class HttpBase extends Step
      */
     protected ?array $inputHeaders = null;
 
+    /**
+     * @var Closure[]
+     */
+    protected array $postBrowserNavigateHooks = [];
+
     /**
      * @param string $method
      * @param array<string, string|string[]> $headers
@@ -127,6 +133,13 @@ public function useInputKeyAsHeaders(string $key): static
         return $this;
     }
 
+    public function postBrowserNavigateHook(Closure $callback): static
+    {
+        $this->postBrowserNavigateHooks[] = $callback;
+
+        return $this;
+    }
+
     /**
      * @return UriInterface|UriInterface[]
      * @throws InvalidArgumentException
@@ -185,10 +198,16 @@ protected function getRequestFromInputUri(UriInterface $uri): RequestInterface
      */
     protected function getResponseFromRequest(RequestInterface $request): ?RespondedRequest
     {
+        $loader = $this->getLoader();
+
+        if (!empty($this->postBrowserNavigateHooks) && $loader->usesHeadlessBrowser()) {
+            $loader->browser()->setTempPostNavigateHooks($this->postBrowserNavigateHooks);
+        }
+
         if ($this->stopOnErrorResponse) {
-            $response = $this->getLoader()->loadOrFail($request);
+            $response = $loader->loadOrFail($request);
         } else {
-            $response = $this->getLoader()->load($request);
+            $response = $loader->load($request);
         }
 
         if ($response !== null && ($response->response->getStatusCode() < 400 || $this->yieldErrorResponses)) {

tests/Loader/Http/HeadlessBrowserLoaderHelperTest.php

@@ -133,3 +133,43 @@ function helper_setUpHeadlessChromeMocks(
 
     expect($invadedBrowserFactory->chromeBinary)->toBe($chromeExecutable);
 });
+
+it('calls the temporary post navigate hooks once', function () {
+    $browserFactoryMock = helper_setUpHeadlessChromeMocks();
+
+    $helper = new HeadlessBrowserLoaderHelper($browserFactoryMock);
+
+    $hook1Called = $hook2Called = $hook3Called = false;
+
+    $helper->setTempPostNavigateHooks([
+        function (Page $page) use (& $hook1Called) {
+            $hook1Called = true;
+        },
+        function (Page $page) use (& $hook2Called) {
+            $hook2Called = true;
+        },
+        function (Page $page) use (& $hook3Called) {
+            $hook3Called = true;
+        },
+    ]);
+
+    $helper->navigateToPageAndGetRespondedRequest(
+        new Request('GET', 'https://www.example.com/foo'),
+        helper_getMinThrottler(),
+    );
+
+    expect($hook1Called)->toBeTrue()
+        ->and($hook2Called)->toBeTrue()
+        ->and($hook3Called)->toBeTrue();
+
+    $hook1Called = $hook2Called = $hook3Called = false;
+
+    $helper->navigateToPageAndGetRespondedRequest(
+        new Request('GET', 'https://www.example.com/foo'),
+        helper_getMinThrottler(),
+    );
+
+    expect($hook1Called)->toBeFalse()
+        ->and($hook2Called)->toBeFalse()
+        ->and($hook3Called)->toBeFalse();
+});

tests/_Integration/Http/HeadlessBrowserTest.php

@@ -10,6 +10,7 @@
 use Crwlr\Crawler\UserAgents\UserAgent;
 use Crwlr\Crawler\UserAgents\UserAgentInterface;
 use Generator;
+use HeadlessChromium\Page;
 use Psr\Log\LoggerInterface;
 use Symfony\Component\DomCrawler\Crawler;
 
@@ -131,3 +132,30 @@ protected function invoke(mixed $input): Generator
         ->and($results[0]->get('printed-cookie'))->toBeString()
         ->and($results[0]->get('printed-cookie'))->toBe('javascriptcookie');
 });
+
+it('gets a cookie that is set via a click, executed via after navigate hook', function () {
+    $crawler = new HeadlessBrowserCrawler();
+
+    $crawler
+        ->input('http://localhost:8000/set-delayed-js-cookie')
+        ->addStep(
+            Http::get()
+                ->postBrowserNavigateHook(function (Page $page) {
+                    $page->mouse()->find('#setCookieButton')->click();
+                }),
+        )
+        ->addStep(new class extends Step {
+            protected function invoke(mixed $input): Generator
+            {
+                yield 'http://localhost:8000/print-cookie';
+            }
+        })
+        ->addStep(Http::get())
+        ->addStep((new GetStringFromResponseHtmlBody())->keepAs('printed-cookie'));
+
+    $results = helper_generatorToArray($crawler->run());
+
+    expect($results)->toHaveCount(1)
+        ->and($results[0]->get('printed-cookie'))->toBeString()
+        ->and($results[0]->get('printed-cookie'))->toBe('jscookie');
+});

tests/_Integration/Server.php

@@ -57,6 +57,10 @@ function getParamAfter(string $route, string $after): string
     return include(__DIR__ . '/_Server/SetCookieJs.php');
 }
 
+if ($route === '/set-delayed-js-cookie') {
+    return include(__DIR__ . '/_Server/SetDelayedCookieJs.php');
+}
+
 if ($route === '/print-cookie') {
     return include(__DIR__ . '/_Server/PrintCookie.php');
 }

tests/_Integration/_Server/SetCookieJs.php

@@ -1,3 +1,4 @@
+<?php setcookie('testcookie', 'foo123'); ?>
 <!doctype html>
 <html lang="de">
 <head><meta charset=utf-8><title>yo</title></head>

tests/_Integration/_Server/SetDelayedCookieJs.php

@@ -0,0 +1,10 @@
+<?php setcookie('testcookie', 'foo123'); ?>
+<!doctype html>
+<html lang="de">
+<head><meta charset=utf-8><title>Hey</title></head>
+<body>
+<div>
+    <button id="setCookieButton" onclick="document.cookie = 'testcookie=jscookie'"></button>
+</div>
+</body>
+</html>