pyterm: add native support

boards/native/Makefile.include

@@ -14,7 +14,19 @@ RESET ?= $(RIOTBOARD)/native/dist/reset.sh
 FLASHER ?=
 FLASHFILE ?= $(ELFFILE)
 
-TERMPROG ?= $(FLASHFILE)
+ifeq (native,$(RIOT_TERMINAL))
+  TERMPROG ?= $(FLASHFILE)
+else
+  TERMFLAGS += -ps $(FLASHFILE)
+  ifeq (1,$(USE_ZEP))
+    ZEP_IP ?= [::1]
+    ZEP_PORT_BASE ?= 17754
+    TERMFLAGS += --process-args '-z $(ZEP_IP):$(ZEP_PORT_BASE)'
+  endif
+  ifneq (,$(ZEP_MAC))
+    TERMFLAGS += --process-args '\-\-eui64=$(ZEP_MAC)'
+  endif
+endif
 
 export VALGRIND ?= valgrind
 export CGANNOTATE ?= cg_annotate
@@ -88,8 +100,12 @@ endif
 LINKFLAGS += -ffunction-sections
 
 # set the tap interface for term/valgrind
-ifneq (,$(filter netdev_tap,$(USEMODULE)))
-  PORT ?= tap0
+PORT ?= tap0
+
+ifeq (native,$(RIOT_TERMINAL))
+  TERMFLAGS += $(PORT)
+else
+  TERMFLAGS += --process-args $(PORT)
 endif
 
 # Configure default eeprom file
@@ -110,7 +126,6 @@ ifneq (,$(filter periph_can,$(FEATURES_USED)))
   TERMFLAGS += $(PERIPH_CAN_FLAGS)
 endif
 
-TERMFLAGS += $(PORT)
 
 ASFLAGS =
 ifeq ($(shell basename $(DEBUGGER)),lldb)

dist/pythonlibs/riotctrl_ctrl/native.py

@@ -15,7 +15,7 @@ class NativeRIOTCtrl(riotctrl.ctrl.RIOTCtrl):
     This works exactly as a normal RIOTCtrl, with the exception that
     `DEBUG_ADAPTER_ID` is set in the environment to the PID of the `native`
     process, whenever a terminal is started. This allows for `reset()` to also
-    work for a the `native` instance.
+    work for a `native` instance.
     """
 
     def _set_debug_adapter_id(self, child):
@@ -27,5 +27,6 @@ def _set_debug_adapter_id(self, child):
     def start_term(self, *args, **kwargs):
         super().start_term(*args, **kwargs)
         for child in psutil.Process(pid=self._term_pid()).children():
-            if self._set_debug_adapter_id(child):
-                break
+            for grandchild in child.children():
+                if self._set_debug_adapter_id(grandchild):
+                    break

dist/pythonlibs/riotctrl_ctrl/requirements.txt

@@ -1,2 +1,3 @@
 psutil
+pyserial
 riotctrl

dist/tools/pyterm/pyterm

@@ -39,6 +39,8 @@ import re
 import codecs
 import platform
 
+from subprocess import Popen, PIPE
+
 try:
     serial.Serial
 except AttributeError:
@@ -107,18 +109,21 @@ class SerCmd(cmd.Cmd):
     port.
     """
 
-    def __init__(self, port=None, baudrate=None, toggle=None, tcp_serial=None,
+    def __init__(self, port=None, baudrate=None, toggle=None,
+                 tcp_serial=None, process=None, process_args=None,
                  confdir=None, conffile=None, host=None, run_name=None,
-                 log_dir_name=None, newline=None, formatter=None,
-                 set_rts=None, set_dtr=None, serprompt=None,
+                 log_dir_name=None, newline=None, formatter=None, set_rts=None,
+                 set_dtr=None, serprompt=None,
                  repeat_command_on_empty_line=defaultrepeat_cmd_empty_line,
                  reconnect=defaultreconnect):
         """Constructor.
 
         Args:
             port (str):         serial port
             baudrate (int):     serial baudrate
-            tcp_serial (iht):   TCP port to connect to (alternatively)
+            tcp_serial (iht):   TCP port to connect to (alterprocessly)
+            process (str):      Program to run as subprocess and connect to its I/O
+            process_args (str): Optional arguments for the subprocess
             confdir (str):      configuration directory
             conffile (str):     configuration file name
             host (str):         local host name
@@ -134,6 +139,14 @@ class SerCmd(cmd.Cmd):
         self.set_rts = set_rts
         self.set_dtr = set_dtr
         self.tcp_serial = tcp_serial
+        self.process = process
+        self.process_args = []
+        if process_args:
+            for arg in process_args:
+                for elem in arg:
+                    for substr in elem.split(' '):
+                        substr = substr.replace('\\-\\-', '--')
+                        self.process_args.append(substr)
         self.configdir = confdir
         self.configfile = conffile
         self.host = host
@@ -204,6 +217,11 @@ class SerCmd(cmd.Cmd):
         """Executed bat program start.
         """
 
+        if self.process and (self.port != defaultport or self.tcp_serial):
+            self.logger.error("Specified a process instance AND a serial "
+                              "port or TCP connection. You probably want "
+                              "to specify only the one or the other.")
+            sys.exit(1)
         # if no serial or TCP is specified use default serial port
         if not self.port and not self.tcp_serial:
             sys.stderr.write("No port specified, using default (%s)!\n"
@@ -253,6 +271,10 @@ class SerCmd(cmd.Cmd):
                 self.logger.error("Something went wrong connecting to "
                                   "localhost:%s" % self.tcp_serial)
                 sys.exit(1)
+        elif self.process:
+            process_call = [self.process] + self.process_args
+            self.logger.debug("Executing process as %s", str(process_call))
+            self.ser = Popen(process_call, stdout=PIPE, stdin=PIPE, stderr=PIPE)
         # otherwise go for the serial port
         elif self.port:
             connected = False
@@ -281,7 +303,6 @@ class SerCmd(cmd.Cmd):
         # commands to the node
         time.sleep(1)
         for command in self.init_cmd:
-            self.logger.debug("WRITE ----->>>>>> '" + command + "'\n")
             self.onecmd(self.precmd(command))
 
         # start serial->console thread
@@ -316,15 +337,15 @@ class SerCmd(cmd.Cmd):
         for tok in line.split(';'):
             tok = self.get_alias(tok)
             if sys.version_info[0] == 2:
-                self.ser.write((tok.strip() + "\n").decode("utf-8").encode("utf-8"))
+                self._write_char((tok.strip() + "\n").decode("utf-8").encode("utf-8"))
             else:
-                self.ser.write((tok.strip() + "\n").encode("utf-8"))
+                self._write_char((tok.strip() + "\n").encode("utf-8"))
 
     def do_help(self, line):
         """Do not use Cmd's internal help function, but redirect to the
         node.
         """
-        self.ser.write("help\n".encode("utf-8"))
+        self._write_char("help\n".encode("utf-8"))
 
     def do_EOF(self, line):
         """Handle EOF (Ctrl+D) nicely."""
@@ -668,11 +689,22 @@ class SerCmd(cmd.Cmd):
             self.ser.setDTR(self.set_dtr)
 
     def _read_char(self):
+        output_stream = self.ser
+        if self.process:
+            output_stream = self.ser.stdout
         # check if serial port can be accessed.
-        sr = codecs.getreader("UTF-8")(self.ser,
+        sr = codecs.getreader("UTF-8")(output_stream,
                                        errors='replace')
         return sr.read(1)
 
+    def _write_char(self, output):
+        input_stream = self.ser
+        if self.process:
+            input_stream = self.ser.stdin
+        input_stream.write(output)
+        if self.process:
+            input_stream.flush()
+
     def _handle_serial_exception(self):
         self.logger.warning("Serial port disconnected, waiting to "
                             "get reconnected...")
@@ -758,7 +790,7 @@ class PytermProt(Protocol):
         if(data.strip() == "/exit"):
             reactor.callLater(2, self.factory.shell.do_PYTERM_exit, data)
         else:
-            self.factory.shell.ser.write(data + "\n")
+            self.factory.shell._write_char(data + "\n")
 
     def sendMessage(self, msg):
         self.transport.writeSomeData("%d#%s\n" % (len(msg), msg))
@@ -815,6 +847,12 @@ if __name__ == "__main__":
                         help="Connect to a TCP port instead of a serial port. "
                         "Format is <hostname>:<port>. If the colon is missing"
                         " host defaults to \"localhost\"")
+    parser.add_argument("-ps", "--process",
+                        help="Start a subprocess and connect pyterm to its "
+                        "stdio.")
+    parser.add_argument("-pa", "--process-args",
+                        help="Adding optional arguments to subprocess",
+                        action="append", nargs='*')
     parser.add_argument("-b", "--baudrate",
                         help="Specifies baudrate for the serial port, default "
                         "is %s" % defaultbaud,
@@ -891,6 +929,7 @@ if __name__ == "__main__":
                         action="store_false",
                         help="Do not try to reconnect when failing on "
                              "connection setup (Default)")
+
     parser.set_defaults(
         repeat_command_on_empty_line=defaultrepeat_cmd_empty_line,
         reconnect=defaultreconnect)
@@ -900,10 +939,10 @@ if __name__ == "__main__":
     if args.noprefix:
         args.format = ""
     myshell = SerCmd(args.port, args.baudrate, args.toggle, args.tcp_serial,
-                     args.directory, args.config, args.host, args.run_name,
-                     args.log_dir_name, args.newline, args.format,
-                     args.set_rts, args.set_dtr, args.prompt,
-                     args.repeat_command_on_empty_line)
+                     args.process, args.process_args, args.directory,
+                     args.config, args.host, args.run_name, args.log_dir_name,
+                     args.newline, args.format, args.set_rts, args.set_dtr,
+                     args.prompt, args.repeat_command_on_empty_line)
     myshell.prompt = ''
 
     try:

examples/gcoap/Makefile

@@ -70,12 +70,7 @@ USE_ZEP ?= 0
 # set the ZEP port for native
 ZEP_PORT_BASE ?= 17754
 ifeq (1,$(USE_ZEP))
-  TERMFLAGS += -z [::1]:$(ZEP_PORT_BASE)
   USEMODULE += socket_zep
-
-  ifneq (,$(ZEP_MAC))
-    TERMFLAGS += --eui64=$(ZEP_MAC)
-  endif
 endif
 
 include $(RIOTBASE)/Makefile.include

examples/gcoap_dtls/Makefile

@@ -82,12 +82,7 @@ USE_ZEP ?= 0
 # set the ZEP port for native
 ZEP_PORT_BASE ?= 17754
 ifeq (1,$(USE_ZEP))
-  TERMFLAGS += -z [::1]:$(ZEP_PORT_BASE)
   USEMODULE += socket_zep
-
-  ifneq (,$(ZEP_MAC))
-    TERMFLAGS += --eui64=$(ZEP_MAC)
-  endif
 endif
 
 include $(RIOTBASE)/Makefile.include

examples/gnrc_border_router/Makefile.native.conf

@@ -34,15 +34,11 @@ endif
 # Configure terminal parameters
 TERMDEPS += host-tools
 TERMPROG_FLAGS = $(FLAGS_EXTRAS) $(ELFFILE) $(IPV6_PREFIX)
+RIOT_TERMINAL ?= native
 TERMPROG ?= sudo $(RIOTTOOLS)/zep_dispatch/start_network.sh $(TERMPROG_FLAGS)
 
 # -z [::1]:$PORT for each ZEP device
 TERMFLAGS ?= $(patsubst %,-z [::1]:%, $(shell seq $(ZEP_PORT_BASE) $(ZEP_PORT_MAX)))
 
-# set optional ZEP l2 address
-ifneq (,$(ZEP_MAC))
-  TERMFLAGS += --eui64=$(ZEP_MAC)
-endif
-
 # native uses $PORT to specify the TAP interface
 PORT ?= $(TAP)

examples/gnrc_networking/Makefile

@@ -47,12 +47,7 @@ USE_ZEP ?= 0
 # set the ZEP port for native
 ZEP_PORT_BASE ?= 17754
 ifeq (1,$(USE_ZEP))
-  TERMFLAGS += -z [::1]:$(ZEP_PORT_BASE)
   USEMODULE += socket_zep
-
-  ifneq (,$(ZEP_MAC))
-    TERMFLAGS += --eui64=$(ZEP_MAC)
-  endif
 endif
 
 # Uncomment the following 2 lines to specify static link lokal IPv6 address

makefiles/tools/serial.inc.mk

@@ -80,4 +80,6 @@ else ifeq (${RIOT_TERMINAL},bootterm)
   TERMPROG = $(RIOTTOOLS)/bootterm/bt
   TERMFLAGS = $(BOOTTERMFLAGS) -a -b $(BAUD) $(PORT)
   TERMDEPS += $(TERMPROG)
+else ifeq (${RIOT_TERMINAL},native)
+  TERMPROG ?= $(ELFFILE)
 endif

tests/core/sched_change_priority/Makefile

@@ -4,9 +4,6 @@ USEMODULE += nice
 USEMODULE += ps
 USEMODULE += shell_cmds_default
 
-# Use a terminal that does not introduce extra characters into the stream.
-RIOT_TERMINAL ?= socat
-
 APP_SHELL_FMT ?= NONE
 
 include $(RIOTBASE)/Makefile.include

tests/net/emcute/Makefile

@@ -6,7 +6,7 @@ export TAP ?= tap0
 
 # use Ethernet as link-layer protocol
 ifeq (native,$(BOARD))
-  TERMFLAGS ?= $(TAP)
+  PORT ?= $(TAP)
 else
   ETHOS_BAUDRATE ?= 115200
   CFLAGS += -DETHOS_BAUDRATE=$(ETHOS_BAUDRATE)

tests/net/gcoap_dns/Makefile

@@ -28,6 +28,9 @@ CFLAGS += -DHAS_SOCK_DNS_MOCK=1
 include $(RIOTBASE)/Makefile.include
 
 ifeq (native,$(BOARD))
+RIOT_TERMINAL=native
+TERMPROG=$(ELFFILE)
+TERMFLAGS=
 test: PORT=
 $(call target-export-variables,test,PORT)
 endif

tests/net/gcoap_fileserver/Makefile

@@ -37,6 +37,7 @@ ifeq (native, $(BOARD))
   USEMODULE += socket_zep
   USEMODULE += socket_zep_hello
   USEMODULE += netdev
+  RIOT_TERMINAL = native
   TERMFLAGS += -z 127.0.0.1:17754 # Murdock has no IPv6 support
   # make sure each instance gets their own fs
   CFLAGS += -DCONFIG_NATIVE_ISOLATE_FS=1

tests/net/gnrc_dhcpv6_client_6lbr/Makefile

@@ -15,6 +15,7 @@ USEMODULE += shell_cmds_default
 
 # use Ethernet as link-layer protocol
 ifeq (native,$(BOARD))
+  RIOT_TERMINAL = native
   TERMFLAGS += -z [::1]:17754
 else
   ETHOS_BAUDRATE ?= 115200

tests/net/gnrc_dhcpv6_relay/Makefile

@@ -12,7 +12,7 @@ USEMODULE += gnrc_netif_single          # Only one interface used and it makes
 
 # use Ethernet as link-layer protocol
 ifeq (native,$(BOARD))
-  TERMFLAGS ?= $(TAP)
+  PORT ?= $(TAP)
 else
   ETHOS_BAUDRATE ?= 115200
   CFLAGS += -DETHOS_BAUDRATE=$(ETHOS_BAUDRATE)

tests/net/gnrc_ipv6_ext/Makefile

@@ -6,7 +6,7 @@ export TAP ?= tap0
 
 # use Ethernet as link-layer protocol
 ifeq (native,$(BOARD))
-  TERMFLAGS ?= $(TAP)
+  PORT ?= $(TAP)
 else
   ETHOS_BAUDRATE ?= 115200
   CFLAGS += -DETHOS_BAUDRATE=$(ETHOS_BAUDRATE)

tests/net/gnrc_ipv6_ext_frag/Makefile

@@ -8,7 +8,7 @@ CFLAGS += -DOUTPUT=TEXT
 CFLAGS += -DTEST_SUITES="gnrc_ipv6_ext_frag"
 
 ifeq (native,$(BOARD))
-  TERMFLAGS ?= $(TAP)
+  PORT ?= $(TAP)
 else
   ETHOS_BAUDRATE ?= 115200
   CFLAGS += -DETHOS_BAUDRATE=$(ETHOS_BAUDRATE)

tests/net/gnrc_ipv6_ext_opt/Makefile

@@ -6,7 +6,7 @@ export TAP ?= tap0
 
 # use Ethernet as link-layer protocol
 ifeq (native,$(BOARD))
-  TERMFLAGS ?= $(TAP)
+  PORT ?= $(TAP)
 else
   ETHOS_BAUDRATE ?= 115200
   CFLAGS += -DETHOS_BAUDRATE=$(ETHOS_BAUDRATE)

tests/net/gnrc_ipv6_nib_dns/Makefile

@@ -10,7 +10,7 @@ USEMODULE += gnrc_ipv6_default
 USEMODULE += gnrc_ipv6_nib_dns
 # use Ethernet as link-layer protocol
 ifeq (native,$(BOARD))
-  TERMFLAGS ?= $(TAP)
+  PORT ?= $(TAP)
 else
   ETHOS_BAUDRATE ?= 115200
   CFLAGS += -DETHOS_BAUDRATE=$(ETHOS_BAUDRATE)

tests/net/gnrc_netif_ieee802154/Makefile

@@ -4,6 +4,7 @@ include ../Makefile.net_common
 
 ifeq (native, $(BOARD))
   USEMODULE += socket_zep
+  RIOT_TERMINAL = native
   TERMFLAGS ?= -z "0.0.0.0:17755,localhost:17754"
   USEMODULE += netdev
   # somehow this breaks the test