From 20ee9f6a3bf93c8596871c3a6a5e25ac43e20108 Mon Sep 17 00:00:00 2001 From: Oldes Huhuman Date: Fri, 27 Dec 2024 00:02:28 +0100 Subject: [PATCH] FEAT: Upgrade WebDriver module to version 0.2.1 --- src/modules/webdriver.reb | 273 +++++++++++++++++++++++++++----------- 1 file changed, 196 insertions(+), 77 deletions(-) diff --git a/src/modules/webdriver.reb b/src/modules/webdriver.reb index 9c77d1ab7b..a3c629a9b6 100644 --- a/src/modules/webdriver.reb +++ b/src/modules/webdriver.reb @@ -2,17 +2,18 @@ Rebol [ Title: "WebDriver (chrome) scheme" Type: module Name: webdriver - Date: 03-Jan-2024 - Version: 0.1.0 + Date: 26-Dec-2024 + Version: 0.2.1 Author: @Oldes Home: https://github.com/Oldes/Rebol-WebDriver Rights: http://opensource.org/licenses/Apache-2.0 Purpose: {Can be used to automate browser sessions.} History: [ 03-Jan-2024 "Oldes" {Initial version} + 25-Dec-2024 "Oldes" {Improvements... WIP} ] Needs: [ - 3.16.0 ;; new map and construction syntax since 3.16.0 + 3.11.0 ;; Minimal Rebol version required by WebScocket module websocket json ] @@ -27,11 +28,17 @@ Rebol [ /Applications/Brave\ Browser.app/Contents/MacOS/Brave\ Browser --remote-debugging-port=9222 ``` + It is also possible to run the browser in a Docker: + ```terminal + docker container run -d -p 9222:9222 zenika/alpine-chrome --no-sandbox --remote-debugging-address=0.0.0.0 --remote-debugging-port=9222 + ``` + Available methods are documented here: https://chromedevtools.github.io/devtools-protocol/ + Or once running browser with the debugging enabled: http://localhost:9222/json/protocol } ] -system/options/log/chrome: 4 +system/options/log/chrome: 1 ;; internal functions.... read-and-wait: function[ @@ -43,12 +50,11 @@ read-and-wait: function[ end: start + time until [ read port - wait [port time] - - process-packets port - + if port? wait [port time][ + process-packets port + ] time: difference end now/precise - time <= 0:0:0 + any [time <= 0 not open? port] ] ] @@ -56,22 +62,157 @@ process-packets: function[ "Process incomming webscocket packets of the webdrive scheme" conn [port!] ][ + ;print "--- process-packets" port: conn/parent ;; outter webdrive scheme ctx: port/extra foreach packet conn/data [ + ;? packet try/with [ packet: decode 'json packet - either packet/id [ - ctx/pending: ctx/pending - 1 + either all [packet/id any [packet/result packet/error]] [ + ;- command response + ;? packet + sys/log/info 'CHROME ["Received response:" as-yellow packet/id] + if packet/error [ + sys/log/error 'CHROME [as-red packet/error/code as-purple packet/error/message] + ] + ;sys/log/info 'CHROME [as-red packet/method mold packet/params] + if ctx/wait-for == packet/id [ + ctx/wait-for: none + ] + ;; Keep all responses in the context (for possible future use) + repend ctx/responses [ + ctx/req/method + packet + ] append port/data packet - ][ port/actor/on-method packet ] + ][ + ;- event notification + repend ctx/events [ + packet/method + packet/params + ] + port/actor/on-method packet + if ctx/wait-for == packet/method [ + ;print "SHOULD AWAKE!" + ctx/wait-for: none + ] + if packet/method == "Inspector.detached" [ + sys/log/info 'CHROME ["Closing connection:" as-red packet/params/reason] + close ctx/page-conn + ctx/page-conn: none + clear ctx/command-que + break + ] + ] ] :print ] clear conn/data ] +page-awake: func[event /local port ctx][ + ;print ["--------------------- page-awake" event/type] + port: event/port + ctx: port/extra + switch event/type [ + wrote [ + read port + return false + ] + read [ + process-packets ctx/page-conn + if ctx/wait-for [ + read port + return false + ] + ] + ] + true +] + ws-decode: :codecs/ws/decode +parse-commands: function[ctx data][ + que: ctx/command-que + parse data [some [ + set url: url! ( + append/only que compose/deep [Page.navigate [url: (url)]] + ) + | opt 'wait set time: [time! | decimal! | integer!] ( + ;- Wait some time while processing incomming messages + append/only que reduce ['wait to time! time] + ) + | 'wait set event: [word! | lit-word!] ( + append/only que reduce ['wait to string! event] + ) + | + set method: word! set params: opt [map! | block!] ( + repend/only que [method params] + ) + ]] + que +] +do-next-command: function[port][ + ;print "---do-next-command" + ctx: port/extra + cmd: take ctx/command-que + unless cmd [return none] + set [method: params:] cmd + either method == 'wait [ + sys/log/info 'CHROME ["WAIT" as-green params] + if time? params [ + read-and-wait any [ctx/page-conn ctx/browser] params + exit + ] + ctx/wait-for: params + read conn: ctx/page-conn + ][ + ;- Send a command with optional options + if block? params [params: make map! reduce/no-set params] + sys/log/info 'CHROME ["Command:" as-red method as-green mold/flat params] + ;; resusing `req` value for all commands as it is just used to form a json anyway + ctx/req/id: ctx/counter: ctx/counter + 1 ;; each command has an unique id + ctx/req/method: method + ctx/req/params: params + write conn: any [ctx/page-conn ctx/browser] ctx/req + + ctx/wait-for: any [ + select [ + Page.navigate "Page.frameStoppedLoading" + Page.close "Inspector.detached" + ] method + ctx/req/id + ] + ] + ;; don't wake up until received responses for all command requests + forever [ + if any [ + none? wait [conn 15] ;; timeout + none? ctx/wait-for ;; not waiting for any specific response + ][ break] + read conn ;; keep reading + ] +] + +init-session: function[port][ + ctx: port/extra + clear ctx/command-que + clear ctx/responses + clear ctx/events + ;; Open a blank page in the browser. + sys/log/info 'CHROME "Opening a new blank tab." + ctx/page-info: decode 'json write ctx/host/json/new [PUT] + ctx/page-conn: conn: open as url! ctx/page-info/webSocketDebuggerUrl + conn/parent: port + unless wait [conn 15] [ + do make error! "Failed to open webSocketDebuggerUrl websocket connection!" + ] + sys/log/info 'CHROME ["Session initialized:" ctx/page-info/id] + port/awake: :page-awake + write port 'Page.enable + port +] + ;- The Chrome scheme --------------------------------------------------------------- sys/make-scheme [ name: 'chrome @@ -79,7 +220,7 @@ sys/make-scheme [ spec: object [title: scheme: ref: host: none port: 9222] actor: [ - open: func [port [port!] /local ctx spec host conn data port-spec][ + open: func [port [port!] /local ctx spec conn data][ spec: port/spec spec/host: any [spec/host "localhost"] spec/port: any [spec/port 9222] @@ -90,10 +231,13 @@ sys/make-scheme [ version: none browser: none counter: 0 - pending: 0 ;; increments when a new method is sent, decremented when response is received - req: #[id: 0 method: none params: #(none)] ;; used to send a command (to avoid cerating a new map) + req: make map! [id: 0 method: none params: none] ;; used to send a command (to avoid cerating a new map) page-info: none ;; holds resolved info from an attached page page-conn: none ;; webscocket connection to an attached page + wait-for: none + command-que: copy [] + responses: copy [] + events: copy [] ] ctx/version: data: try/with [ @@ -108,110 +252,85 @@ sys/make-scheme [ conn/parent: port wait [conn 15] sys/log/more 'CHROME "Browser connection opened." - port + + init-session port ] open?: func[port /local ctx][ all [ ctx: port/extra any [ctx/browser ctx/page-conn] + true ] ] close: func[port /local ctx][ ctx: port/extra - if ctx/port-conn [ - try [close ctx/port-conn wait [ctx/page-conn 1]] - ctx/port-conn: ctx/port-info: none - + clear ctx/command-que + if ctx/page-conn [ + write port 'Page.close + ctx/page-info: none ] if ctx/browser [ - try [close ctx/browser wait [ctx/browser 1]] + close ctx/browser ctx/browser: none ] port ] - write: func[port data /local ctx url time method params conn][ + write: func[port data /local ctx url time method params conn pos p][ unless block? data [data: reduce [data]] - sys/log/info 'CHROME ["WRITE:" as-green mold/flat data] + sys/log/debug 'CHROME ["WRITE:" as-green mold/flat data] + + ctx: port/extra clear port/data - ctx: port/extra either open? ctx/browser [ - parse data [some [ - - set url: url! ( - ;- Open a new target (page) - try/with [ - ctx/page-info: decode 'json write join ctx/host/json/new? url [PUT] - ;?? ctx/page-info - append port/data ctx/page-info - - ctx/page-conn: conn: open as url! ctx/page-info/webSocketDebuggerUrl - ;conn/awake: :ws-web-awake - conn/parent: port - wait [conn 15] - conn: none - ] :print - ) - | set time: [time! | decimal! | integer!] ( - ;- Wait some time while processing incomming messages - time: to time! time - sys/log/info 'CHROME ["WAIT" as-green time] - read-and-wait any [ctx/page-conn ctx/browser] time - ) - | - set method: word! set params: opt [map! | block!] ( - ;- Send a command with optional options - if block? params [params: make map! reduce/no-set params] - sys/log/info 'CHROME ["Command:" as-red method as-green mold/flat/part params 100] - ;; resusing `req` value for all commands as it is just used to form a json anyway - ctx/req/id: ctx/counter: ctx/counter + 1 ;; each command has an unique id - ctx/req/method: method - ctx/req/params: params - ctx/pending: ctx/pending + 1 - write conn: any [ctx/page-conn ctx/browser] ctx/req - ;; don't wake up until received responses for all command requests - forever [ - ;@@TODO: handle the timeout to awoid infinite loop! - wait [conn 15] ;; wait for any events - process-packets conn ;; process incomming websocket messages - if ctx/pending <= 0 [break] ;; exit the loop if there are no pending requests - read conn ;; keep reading - ] - ) - ]] + unless ctx/page-conn [ + ;print "---------------------------" + init-session port + ] + parse-commands ctx data + while [not empty? ctx/command-que][ + do-next-command port + ] either 1 = length? port/data [first port/data][port/data] ][ sys/log/error 'CHROME "Not open!"] ] read: func[port /local ctx conn packet][ - ;; waits for any number of incomming messages + ;; waits for any number of incomming messagesto if all [ ctx: port/extra conn: any [ctx/page-conn ctx/browser] ][ - clear port/data read conn - wait [conn 1] ;; don't wait more then 1 second if there are no incomming messages - process-packets conn + ;wait [conn 1] ;; don't wait more then 1 second if there are no incomming messages + ;process-packets conn ] port/data ] pick: func[port value /local result][ ;; just a shortcut to get a single result direcly - unless block? value [value: reduce [value]] - result: write port value - if block? result [result: last result] - result/result + all [ + object? port/extra + select/last port/extra/responses :value + ] ] - on-method: func[packet][ + on-method: func[packet /local verbose][ ;; this function is supposed to be user defined and used to process incomming messages ;; in this case it just prints its content... - sys/log/info 'CHROME [as-red packet/method mold packet/params] + verbose: system/options/log/chrome + case [ + verbose > 1 [ + sys/log/debug 'CHROME ["Event:" as-yellow packet/method mold packet/params] + ] + verbose > 0 [ + sys/log/info 'CHROME ["Event:" as-yellow packet/method] + ] + ] ] ] ]