Merge branch 'master' of github.com:WPO-Foundation/wptagent

README.md

@@ -78,6 +78,7 @@ run the agent in a docker container.
     * none - Disable traffic-shaping (i.e. when root is not available).
     * netem,\<interface\> - Use NetEm for bridging rndis traffic (specify outbound interface).  i.e. --shaper netem,eth0
     * remote,\<server\>,\<down pipe\>,\<up pipe\> - Connect to the remote server over ssh and use pre-configured dummynet pipes (ssh keys for root user should be pre-authorized).
+    * chrome - Use Chrome's dev tools traffic-shaping. Only supports Chromium browsers and should be used as a last resort.
 
 ### Android testing options
 * **--android** : Run tests on an attached android device.

internal/android_browser.py

@@ -203,9 +203,10 @@ def on_start_recording(self, task):
             if version is not None:
                 task['page_data']['browserVersion'] = version
                 task['page_data']['browser_version'] = version
-            if not self.job['shaper'].configure(self.job, task):
-                task['error'] = "Error configuring traffic-shaping"
-                task['page_data']['result'] = 12999
+            if not self.job['dtShaper']:
+                if not self.job['shaper'].configure(self.job, task):
+                    task['error'] = "Error configuring traffic-shaping"
+                    task['page_data']['result'] = 12999
             if self.tcpdump_enabled:
                 self.adb.start_tcpdump()
             if self.video_enabled and not self.job['disable_video']:

internal/desktop_browser.py

@@ -429,8 +429,9 @@ def on_start_recording(self, task):
             return
         import psutil
         if task['log_data']:
-            if not self.job['shaper'].configure(self.job, task):
-                self.task['error'] = "Error configuring traffic-shaping"
+            if not self.job['dtShaper']:
+                if not self.job['shaper'].configure(self.job, task):
+                    self.task['error'] = "Error configuring traffic-shaping"
             self.cpu_start = psutil.cpu_times()
             self.recording = True
             ver = platform.uname()

internal/devtools.py

@@ -220,6 +220,7 @@ def connect(self, timeout):
                                 try:
                                     self.websocket = DevToolsClient(websocket_url)
                                     self.websocket.connect()
+                                    self.job['shaper'].set_devtools(self)
                                     ret = True
                                 except Exception as err:
                                     logging.exception("Connect to dev tools websocket Error: %s", err.__str__())
@@ -246,11 +247,35 @@ def connect(self, timeout):
         self.profile_end('connect')
         return ret
 
+    def _to_int(self, s):
+        return int(re.search(r'\d+', str(s)).group())
+
+    def enable_shaper(self, target_id=None):
+        """Enable the Chromium dev tools traffic shaping"""
+        if self.job['dtShaper']:
+            in_Bps = -1
+            if 'bwIn' in self.job:
+                in_Bps = (self._to_int(self.job['bwIn']) * 1000) / 8
+            out_Bps = -1
+            if 'bwOut' in self.job:
+                out_Bps = (self._to_int(self.job['bwOut']) * 1000) / 8
+            rtt = 0
+            if 'latency' in self.job:
+                rtt = self._to_int(self.job['latency'])
+            self.send_command('Network.emulateNetworkConditions', {
+                'offline': False,
+                'latency': rtt,
+                'downloadThroughput': in_Bps,
+                'uploadThroughput': out_Bps
+                }, wait=True, target_id=target_id)
+
     def enable_webkit_events(self):
         if self.is_webkit:
             self.send_command('Inspector.enable', {})
             self.send_command('Network.enable', {})
             self.send_command('Runtime.enable', {})
+            self.job['shaper'].apply()
+            self.enable_shaper()
             if self.headers:
                 self.send_command('Network.setExtraHTTPHeaders', {'headers': self.headers})
             if len(self.workers):
@@ -281,6 +306,7 @@ def prepare_browser(self):
 
     def close(self, close_tab=True):
         """Close the dev tools connection"""
+        self.job['shaper'].set_devtools(None)
         if self.websocket:
             try:
                 self.websocket.close()
@@ -1185,6 +1211,8 @@ def enable_target(self, target_id=None):
             self.send_command('Log.enable', {}, target_id=target_id)
             self.send_command('Log.startViolationsReport', {'config': [{'name': 'discouragedAPIUse', 'threshold': -1}]}, target_id=target_id)
             self.send_command('Audits.enable', {}, target_id=target_id)
+            self.job['shaper'].apply(target_id=target_id)
+            self.enable_shaper(target_id=target_id)
             if self.headers:
                 self.send_command('Network.setExtraHTTPHeaders', {'headers': self.headers}, target_id=target_id, wait=True)
             if 'user_agent_string' in self.job:
@@ -1491,8 +1519,8 @@ def process_target_event(self, event, msg):
                 target = msg['params']['targetInfo']
                 if 'type' in target and target['type'] == 'service_worker':
                     self.workers.append(target)
-                    if self.recording:
-                        self.enable_target(target['targetId'])
+                if self.recording:
+                    self.enable_target(target['targetId'])
                 self.send_command('Runtime.runIfWaitingForDebugger', {},
                                   target_id=target['targetId'])
         if event == 'receivedMessageFromTarget' or event == 'dispatchMessageFromTarget':

internal/devtools_browser.py

@@ -750,7 +750,7 @@ def run_lighthouse_test(self, task):
         """Run a lighthouse test against the current browser session"""
         task['lighthouse_log'] = ''
         if 'url' in self.job and self.job['url'] is not None and not self.is_webkit:
-            if not self.job['lighthouse_config']:
+            if not self.job['lighthouse_config'] and not self.job['dtShaper']:
                 self.job['shaper'].configure(self.job, task)
             output_path = os.path.join(task['dir'], 'lighthouse.json')
             json_file = os.path.join(task['dir'], 'lighthouse.report.json')

internal/safari_ios.py

@@ -843,8 +843,9 @@ def on_start_recording(self, task):
         self.flush_messages()
         self.enable_safari_events()
         if self.task['log_data']:
-            if not self.job['shaper'].configure(self.job, task):
-                self.task['error'] = "Error configuring traffic-shaping"
+            if not self.job['dtShaper']:
+                if not self.job['shaper'].configure(self.job, task):
+                    self.task['error'] = "Error configuring traffic-shaping"
             if self.bodies_zip_file is not None:
                 self.bodies_zip_file.close()
                 self.bodies_zip_file = None

internal/traffic_shaping.py

@@ -24,6 +24,8 @@ def __init__(self, options, root_path):
         if shaper_name is not None:
             if shaper_name == 'none':
                 self.shaper = NoShaper()
+            elif shaper_name == 'chrome':
+                self.shaper = ChromeShaper()
             elif shaper_name[:5] == 'netem':
                 parts = shaper_name.split(',')
                 if_out = parts[1].strip() if len(parts) > 1 else None
@@ -111,6 +113,20 @@ def configure(self, job, task):
             job['interface'] = self.shaper.interface
         return ret
 
+    def set_devtools(self, devtools):
+        """Configure the devtools interface for the shaper (Chrome-only)"""
+        try:
+            self.shaper.set_devtools(devtools)
+        except Exception:
+            logging.exception('Error setting shaper devtools interface')
+        return
+
+    def apply(self, target_id=None):
+        """Apply the traffic-shaping for Chrome"""
+        try:
+            self.shaper.apply(target_id)
+        except Exception:
+            logging.exception('Error applying traffic shaping')
 
 #
 # NoShaper
@@ -137,6 +153,64 @@ def configure(self, in_bps, out_bps, rtt, plr, shaperLimit):
         if in_bps > 0 or out_bps > 0 or rtt > 0 or plr > 0 or shaperLimit > 0:
             return False
         return True
+
+    def set_devtools(self, devtools):
+        """Stub for configuring the devtools interface"""
+        return
+
+    def apply(self, target_id):
+        """Stub for applying Chrome traffic-shaping"""
+        return
+#
+# ChromeShaper
+#
+class ChromeShaper(object):
+    """Allow resets but fail any explicit shaping"""
+    def __init__(self):
+        self.interface = None
+        self.devtools = None
+        self.rtt = 0
+        self.in_Bps = -1
+        self.out_Bps = -1
+
+    def install(self):
+        """Install and configure the traffic-shaper"""
+        return True
+
+    def remove(self):
+        """Uninstall traffic-shaping"""
+        return True
+
+    def reset(self):
+        """Disable traffic-shaping"""
+        self.rtt = 0
+        self.in_Bps = -1
+        self.out_Bps = -1
+        self.apply()
+        return True
+
+    def configure(self, in_bps, out_bps, rtt, plr, shaperLimit):
+        """Enable traffic-shaping"""
+        self.rtt = rtt
+        self.in_Bps = in_bps / 8
+        self.out_Bps = out_bps / 8
+        self.apply()
+        return True
+
+    def set_devtools(self, devtools):
+        """Stub for configuring the devtools interface"""
+        self.devtools = devtools
+
+    def apply(self, target_id=None):
+        """Stub for applying Chrome traffic-shaping"""
+        if self.devtools is not None:
+            self.devtools.send_command('Network.emulateNetworkConditions', {
+                'offline': False,
+                'latency': self.rtt,
+                'downloadThroughput': self.in_Bps,
+                'uploadThroughput': self.out_Bps
+                }, wait=True, target_id=target_id)
+        return
 
 #
 # winshaper
@@ -180,6 +254,14 @@ def configure(self, in_bps, out_bps, rtt, plr, shaperLimit):
                             'inbuff={0:d}'.format(int(self.in_buff)),
                             'outbuff={0:d}'.format(int(self.out_buff))])
 
+    def set_devtools(self, devtools):
+        """Stub for configuring the devtools interface"""
+        return
+
+    def apply(self, target_id):
+        """Stub for applying Chrome traffic-shaping"""
+        return
+
 #
 # Dummynet
 #
@@ -273,6 +355,14 @@ def configure(self, in_bps, out_bps, rtt, plr, shaperLimit):
                self.ipfw(in_queue_command) and\
                self.ipfw(out_queue_command)
 
+    def set_devtools(self, devtools):
+        """Stub for configuring the devtools interface"""
+        return
+
+    def apply(self, target_id):
+        """Stub for applying Chrome traffic-shaping"""
+        return
+
 #
 # MacDummynet - Dummynet through pfctl
 #
@@ -375,6 +465,14 @@ def configure(self, in_bps, out_bps, rtt, plr, shaperLimit):
         return self.dnctl(in_command) and\
                self.dnctl(out_command)
 
+    def set_devtools(self, devtools):
+        """Stub for configuring the devtools interface"""
+        return
+
+    def apply(self, target_id):
+        """Stub for applying Chrome traffic-shaping"""
+        return
+
 #
 # RemoteDummynet - Remote PC running dummynet with pre-configured pipes
 #
@@ -411,6 +509,14 @@ def remove(self):
         """Uninstall traffic-shaping"""
         return True
 
+    def set_devtools(self, devtools):
+        """Stub for configuring the devtools interface"""
+        return
+
+    def apply(self, target_id):
+        """Stub for applying Chrome traffic-shaping"""
+        return
+
 #
 # netem
 #
@@ -515,3 +621,11 @@ def configure_interface(self, interface, bps, latency, plr, shaperLimit):
         args = self.build_command_args(interface, bps, latency, plr, shaperLimit)
         logging.debug(' '.join(args))
         return subprocess.call(args) == 0
+
+    def set_devtools(self, devtools):
+        """Stub for configuring the devtools interface"""
+        return
+
+    def apply(self, target_id):
+        """Stub for applying Chrome traffic-shaping"""
+        return

internal/webpagetest.py

@@ -625,6 +625,7 @@ def get_test(self, browsers):
                     if 'video' not in job:
                         job['video'] = bool('Capture Video' in job and job['Capture Video'])
                     job['keepvideo'] = bool('keepvideo' in job and job['keepvideo'])
+                    job['dtShaper'] = bool('dtShaper' in job and job['dtShaper'])
                     job['disable_video'] = bool(not job['video'] and
                                                 'disable_video' in job and
                                                 job['disable_video'])