chore(chrome): fix frame establishing

spider-rs · Sep 7, 2024 · dca3d09 · dca3d09
1 parent 949b91d
commit dca3d09
Show file tree

Hide file tree

Showing 10 changed files with 43 additions and 26 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/examples/Cargo.toml b/examples/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_examples"
-version = "2.2.16"
+version = "2.2.18"
 authors = [
     "j-mendez <[email protected]>",
 ]

diff --git a/spider/Cargo.toml b/spider/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.2.16"
+version = "2.2.18"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider/src/packages/scraper/element_ref/mod.rs b/spider/src/packages/scraper/element_ref/mod.rs
@@ -58,7 +58,7 @@ impl<'a> ElementRef<'a> {
         };
         let mut buf = Vec::new();
         match serialize(&mut buf, self, opts) {
-            _ => ()
+            _ => (),
         };
         crate::page::encode_bytes_from_language(&buf, "")
     }

diff --git a/spider/src/packages/scraper/html/mod.rs b/spider/src/packages/scraper/html/mod.rs
@@ -105,7 +105,7 @@ impl Html {
         };
         let mut buf = Vec::new();
         match serialize(&mut buf, self, opts) {
-            _ => ()
+            _ => (),
         };
         crate::page::encode_bytes_from_language(&buf, "")
     }

diff --git a/spider/src/utils/mod.rs b/spider/src/utils/mod.rs
@@ -841,24 +841,40 @@ pub async fn fetch_page_html_chrome_base(
             if !page_set {
                 // used for smart mode re-rendering direct assigning html
                 if content {
-                    match page.set_content(source).await {
-                        Ok(p) => {
-                            valid = true;
-                            p
+                    match page.mainframe().await {
+                        Ok(frame) => {
+                            match page.execute(chromiumoxide::cdp::browser_protocol::page::SetDocumentContentParams {
+                                frame_id: frame.unwrap_or_default(),
+                                html: source.to_string()
+                            }).await {
+                                Ok(_p) => {
+                                    valid = true;
+                                    page
+                                }
+                                _ => page,
+                            }
+                        }
+                        _ => {
+                            match page.set_content(source).await {
+                                Ok(p) => {
+                                    valid = true;
+                                    p
+                                }
+                                _ => page,
+                            }
                         }
-                        _ => page,
                     }
                 } else {
-                    match perform_chrome_http_request(&page, source).await {
-                        Ok(chqr) => {
+                    match page.goto(source).await {
+                        Ok(p) => {
                             valid = true;
-                            chrome_http_req_res = chqr;
+                            p
                         }
                         Err(e) => {
                             log("HTTP Error: ", e.to_string());
+                            page
                         }
-                    };
-                    page
+                    }
                 }
             } else {
                 page
@@ -975,7 +991,10 @@ pub async fn fetch_page_html_chrome_base(
         .await;
     }
 
-    if !page_set && cfg!(feature = "cache_chrome_hybrid") {
+    if !page_set
+        && cfg!(feature = "cache_chrome_hybrid")
+        && chrome_http_req_res.status_code.is_success()
+    {
         match url::Url::parse(source) {
             Ok(u) => {
                 let http_response = HttpResponse {

diff --git a/spider/src/website.rs b/spider/src/website.rs
@@ -375,7 +375,6 @@ async fn setup_chrome_interception_base(
     }
 }
 
-
 /// Setup interception for chrome request. This does nothing without the 'chrome_intercept' flag.
 #[cfg(all(feature = "chrome", not(feature = "chrome_intercept")))]
 async fn setup_chrome_interception_base(
@@ -385,10 +384,9 @@ async fn setup_chrome_interception_base(
     _ignore_visuals: bool,
     _host_name: &str,
 ) -> Option<tokio::task::JoinHandle<()>> {
-   None
+    None
 }
 
-
 /// Semaphore low priority tasks to run
 #[cfg(not(feature = "cowboy"))]
 async fn run_task<F, Fut>(

diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.2.16"
+version = "2.2.18"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_utils/Cargo.toml b/spider_utils/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.2.16"
+version = "2.2.18"
 authors = [
     "j-mendez <[email protected]>"
 ]

diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.2.16"
+version = "2.2.18"
 authors = [
     "j-mendez <[email protected]>"
 ]