06-single-hop-udp: Automate all iot-lab tests.

06-single-hop-udp/common.py

@@ -1,23 +1,25 @@
+from collections import namedtuple
 import argparse
 
 from testutils import Board
 from mixins import GNRC, GNRC_UDP,PktBuf
 from time import sleep
 
 
+Result = namedtuple("Result", "packet_loss src_buf_empty dst_buf_empty")
+
 #Declare node
 class SixLoWPANNode(Board, GNRC, GNRC_UDP, PktBuf):
     pass
 
 
-def print_results(results):
-    packet_losses = [results[i][0] for i in range(len(results))]
+def print_results(results_iter):
+    results = list(results_iter)
+    print("")
     print("Summary of {packet losses, source pktbuf sanity, dest pktbuf sanity}:")
-    for i in range(len(results)):
-        print("Run {}: {} {} {}".format(i+1, packet_losses[i], results[i][1], results[i][2]))
+    for spec, r in results:
+        print("Run {}: {}".format(spec, r))
     print("")
-    print("Average packet losses: {}".format(sum(packet_losses)/len(packet_losses)))
-
 
 def udp_send(source, dest, ip_dest, port, count, payload_size, delay):
     source.reboot()
@@ -28,7 +30,7 @@ def udp_send(source, dest, ip_dest, port, count, payload_size, delay):
     packet_loss = dest.udp_server_check_output(count, delay)
     dest.udp_server_stop()
 
-    return packet_loss, source.is_empty(), dest.is_empty()
+    return Result(packet_loss, source.is_empty(), dest.is_empty())
 
 
 argparser = argparse.ArgumentParser()

06-single-hop-udp/tasks.py

@@ -0,0 +1,93 @@
+#! /usr/bin/env python3
+# Copyright (C) 2018 Freie Universität Berlin
+#
+# This file is subject to the terms and conditions of the GNU Lesser
+# General Public License v2.1. See the file LICENSE in the top level
+# directory for more details.
+
+import sys
+import os
+
+import tqdm
+
+class TestSpec:
+    PORT = 1337
+    COUNT = 1000
+    PAYLOAD_SIZE = 1024
+    DELAY = 1000   # ms
+    ERROR_TOLERANCE = 5     # %
+    ADDRESS = "dest"
+
+    def __init__(self, **kwargs):
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+
+    def check(self, result):
+        """Return the result if it is valid, or fail."""
+        assert result.packet_loss < self.ERROR_TOLERANCE
+        assert result.src_buf_empty
+        assert result.dst_buf_empty
+
+        return result
+
+
+class GhostNode:
+    def get_ip_addr(self):
+        return "fe80::bd:b7ec"
+
+
+SPECS = {'normal': TestSpec(),
+         'compression': TestSpec(PORT=61616),
+         'nonexistant': TestSpec(ERROR_TOLERANCE=100, PAYLOAD_SIZE=8,
+                                 ADDRESS="ghost", DELAY=0),
+         'empty': TestSpec(PAYLOAD_SIZE=0, COUNT=10, INTERVAL=100),
+        }
+
+
+def run_spec(nodes, spec):
+    dest_ip = nodes[spec.ADDRESS].get_ip_addr()
+
+    return udp_send(nodes['source'], nodes['dest'], dest_ip, spec.PORT,
+                    spec.COUNT, spec.PAYLOAD_SIZE, spec.DELAY)
+
+
+def run_tasks(riotbase, runs=1):
+    os.chdir(os.path.join(riotbase, "tests/gnrc_udp"))
+
+    node_specs = [IoTLABNode(site="lille", extra_modules=["gnrc_pktbuf_cmd"]),
+                  IoTLABNode(site="lille", extra_modules=["gnrc_pktbuf_cmd"])]
+
+    with IoTLABExperiment("RIOT-release-test-06-01", node_specs) as exp:
+        addrs = exp.nodes_addresses
+        iotlab_cmd = "make IOTLAB_NODE={} BOARD=iotlab-m3 term"
+        nodes = {'source': SixLoWPANNode(iotlab_cmd.format(addrs[0])),
+                 'dest': SixLoWPANNode(iotlab_cmd.format(addrs[1])),
+                 'ghost': GhostNode()
+                }
+        results = []
+
+        spec_progress = tqdm.tqdm(SPECS.items(), unit="Specs")
+
+        results = ((spec_name, spec_progress.set_description(
+                                "Spec: {}".format(spec_name))
+                              or spec.check(run_spec(nodes, spec)))
+                   for spec_name, spec in spec_progress)
+
+        print_results(results)
+
+if __name__ == "__main__":
+    sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                                 "../", "testutils"))
+    from iotlab import IoTLABNode, IoTLABExperiment, IoTLABExperimentError
+    from common import argparser, SixLoWPANNode, udp_send, print_results
+
+    args = argparser.parse_args()
+
+    try:
+        run_tasks(**vars(args))
+    except IoTLABExperimentError as e:
+        print(str(e))
+        print("Can't start experiment")
+    except AssertionError as e:
+        print("FAILED")
+        print(str(e))

testutils/iotlab.py

@@ -2,6 +2,7 @@
 import json
 import logging
 import subprocess
+import contextlib
 
 
 class IoTLABNode(object):
@@ -61,7 +62,7 @@ class IoTLABExperimentError(Exception):
     pass
 
 
-class IoTLABExperiment(object):
+class IoTLABExperiment(contextlib.AbstractContextManager):
     def __init__(self, name, nodes):
         self.nodes = nodes
         self.name = name
@@ -92,6 +93,9 @@ def __repr__(self):
         return "IoTLABExperiment(name={}, exp_id={})".format(self.name,
                                                              self.exp_id)
 
+    def __exit__(self, *exc):
+        self.stop()
+
     def stop(self):
         if self.exp_id is not None:
             subprocess.check_call(['iotlab-experiment', 'stop',

testutils/mixins.py

@@ -1,4 +1,5 @@
 import pexpect
+import tqdm
 
 
 class GNRC:
@@ -64,7 +65,7 @@ def udp_server_stop(self):
 
     def udp_server_check_output(self, count, delay_ms):
         packets_lost = 0
-        for i in range(count):
+        for i in tqdm.trange(count, desc="Received", unit="packet"):
             exp = self.pexpect.expect([
                    r"Packets received: \d+",
                    r"PKTDUMP: data received:\n"
@@ -96,7 +97,7 @@ def udp_send(self, dest_addr, port, payload, count=1, delay_ms=1000):
             bytes = payload
         except ValueError:
             bytes = len(payload)
-        for i in range(count):
+        for i in tqdm.trange(count, desc="Sent", unit="packet"):
             exp = self.pexpect.expect([
                     "Success: sent {} byte\(s\) to \[{}\]:{}".format(
                         bytes, dest_addr, port),