Use JS response when needed

resources/lang/en.json

@@ -9,6 +9,7 @@
     "failed.content.broken_images": "The page contains broken images. These images were found: :actualValue.",
     "failed.content.broken_links": "The page contains broken links. These links were found: :actualValue.",
     "failed.content.length": "The content is :actualValue characters long. It should be at least :expectedValue characters long.",
+    "failed.content.length.parse": "We were unable to parse the content of this page, please try again.",
     "failed.content.mixed_content": "The page contains links to insecure addresses, while it should not. These links were found :actualValue.",
     "failed.content.multiple_h1": "The page contains multiple h1 tags, while it should not. These tags were found :actualValue.",
     "failed.content.no_heading": "The page does not contain any h1 tag, while it should.",

src/Checks/Content/ContentLengthCheck.php

@@ -38,7 +38,7 @@ public function check(Response $response, Crawler $crawler): bool
             return true;
         }
 
-        $content = $this->getContentToValidate($response);
+        $content = $this->getContentToValidate($response, $crawler);
 
         if (! $content) {
             return true;
@@ -47,15 +47,34 @@ public function check(Response $response, Crawler $crawler): bool
         return $this->validateContent($content);
     }
 
-    public function getContentToValidate(Response $response): ?string
+    public function getContentToValidate(Response $response, Crawler $crawler): ?string
     {
         $url = $response->transferStats->getHandlerStats()['url'];
 
-        $readability = new Readability($response->body(), $url);
+        $body = $response->body();
+
+        if ($this->useJavascript) {
+            $body = $crawler->filter('body')->html();
+        }
+
+        $readability = new Readability($body, $url);
 
         $readability->init();
 
-        return $readability->getContent()->textContent ?? null;
+        $textContent = $readability->getContent()->textContent;
+
+        /**
+         * This is a fallback for when Readability is unable to parse the content.
+         * Sometimes it happens when scanning a JavaScript rendered page, that
+         * we don't get a proper response. In that case we just return null.
+         * @todo we should check if we can improve this.
+         */
+        if ($textContent == 'Sorry, Readability was unable to parse this page for content.') {
+            $this->failureReason = __('failed.content.length.parse');
+            return null;
+        }
+
+        return $textContent ?? null;
     }
 
     public function validateContent(string|array $content): bool