some restructuring of debug libs; updates #1303

.gitignore

@@ -1,9 +1,9 @@
 #network protocol output files
 software/common/generated_libs/protocols/*.c
 software/common/generated_libs/protocols/*.h
-software/utils/debug/lib/protocols/*
 software/gui/src/protocols/*.cc
 software/gui/src/protocols/*.h
+software/utils/debug/protocols/*
 
 build/
 build_autotests/
@@ -58,5 +58,5 @@ fp-info-cache
 *-cache.lib
 
 #performance test data
+local_data/
 test_data/
-device_list.*

software/common/common.sh

@@ -126,7 +126,7 @@ update_platformio() {
 
 generate_network_protocols() {
   PROTOCOLS_DIR=generated_libs/protocols
-  PYTHON_LIB_PATH=../utils/debug/lib/protocols
+  PYTHON_LIB_PATH=../utils/debug/protocols
   GUI_LIB_PATH=../gui/src/protocols
   NANOPB_PATH=$(pip3 show nanopb | awk '{ if($1 == "Location:") print $2}')/nanopb/generator
 

software/controller/controller.sh

@@ -84,6 +84,9 @@ The following options are available:
   unit        Builds and runs unit tests only (and generates coverage reports)
                 <name>  - run specific unit test, may include wildcards, i.e. '*debug*'
                 [-o]    - open coverage report in browser when done
+  integrate   Run integration tests
+                all [delay_time] - run all integration tests, pause for [delay time] in between
+                <test_name> [parameters...] - run specific integration test with parameters
   cov_upload  Upload coverage reports to Codecov server
   help/-h     Display this help info
 EOF
@@ -235,15 +238,15 @@ print_device_info() {
   if [ ! -z "$SN" ]
   then
     echo "SN has been defined in environment, will be deploying to the following device:"
-    ../utils/debug/debug_cli.py -c "devices find $SN"
+    ../utils/debug/debug.sh -c "device find $SN"
   fi
 }
 
 # get_hla_serial <alias>
 # prints ST-Link serial number by defined alias
 get_hla_serial() {
   device_alias="$1"
-  ../utils/debug/debug_cli.py -c "devices find $device_alias h"
+  ../utils/debug/debug.sh -c "device find $device_alias h"
 }
 
 # build <target_name>
@@ -483,7 +486,7 @@ elif [ "$1" == "run" ]; then
 elif [ "$1" == "debug" ]; then
   shift
   pushd ../utils/debug
-  ./debug_cli.py "$@"
+  ./debug.sh "$@"
   popd
 
   exit $EXIT_SUCCESS
@@ -512,7 +515,7 @@ elif [ "$1" == "integrate" ]; then
 # DEVICES #
 ###########
 elif [ "$1" == "devices" ]; then
-  ../utils/debug/debug_cli.py -c "devices list"
+  ../utils/debug/debug.sh -c "device list"
   exit $EXIT_SUCCESS
 
 ################

software/controller/platformio/README.md

@@ -4,35 +4,24 @@ The [build_config](build_config) directory contains the build configuration.
 
 What follows is the configuration for the deployment scripts.
 
-## Deployment script
-
-By default, controller software and integration tests will deploy to any one STM32 that is hooked up to your machine.
-
-The [deploy.sh](deploy.sh) script provides a convenient way to deploy to multiple connected STM devices,
-addressing them by user-defined aliases. In order for this to work, you need to modify the Platformio's STLink script,
-find out the serial number of your Nucleo and add it to the list of aliases.
-
-The script provides its own help on syntax.
-
-The script is also used by integration test tools, so it is a good idea to configure it correctly.
-
-The script is further abstracted by the [../controller.sh](../controller.sh) utility, which will deploy the default
-release firmware to the controller with simplified syntax.
-
 ## Modifying STLink script
 
-For the above script to work, a file in your local platformio configuration found here:
+In order to deploy to multiple STM32 devices on the same system, a file in your local platformio configuration found here:
 `$HOME/.platformio/packages/tool-openocd/scripts/interface/stlink.cfg`
 must be replaced with the [`stlink.cfg`](stlink.cfg) provided here.
 
-> TODO: make a script to patch or replace this file
+This patching is now handled by running `controller.sh patch_ocd`.
 
 This solution is based on a post on the platformio forums
 [here](https://community.platformio.org/t/choosing-stlink-v2-programmer/10716).
 
 The post is rather old, so the TCL syntax had to be modified to make it work.
 
-## Finding your Nucleo's serial number
+### Finding your Nucleo's serial number
+
+Association of STM32 HLA serial number to prototype alias ID is now managed by the python and bash utility scripts. Use `./controller.sh devices` or use the `device` command in the debug interface for more details. The manifest is maintained on Google Drive in a spreadsheet under `Testing`.
+
+The following are instructions for how to do this manually.
 
 To deploy to a particular STM32 boards connected to your system, first find out the serial numbers of your connected Nucleos.
 
@@ -43,29 +32,9 @@ E: ID_SERIAL_SHORT=066FFF303435554157105014
 E: ID_SERIAL_SHORT=066BFF303435554157104916
 ```
 
-On Windows, you can use the STLink utility provided by ST Micro. The serial number shows up in the log when you connect.
-Possibly you might need to use this to update the firmware too. Close the utility after you have what you need.
+On Windows, you can use the STLink utility provided by ST Micro. The serial number shows up in the log when you connect. Possibly you might need to use this to update the firmware too. Close the utility after you have what you need.
 
 With the STLink script modified, you can now deploy firmware like this:
 ```
 SERIAL=066BFF303435554157104916 TEST=TEST_IDLE pio run -e integration-test -t upload
 ```
-
-However, this can get tedious, so we recommend...
-
-## Making an alias
-
-The [device_lookup_table.txt](device_lookup_table.txt) file contains a list of aliases.
-
-The list will look something like this:
-```
-ALIAS  NUCLEO_SERIAL                COMMENT
-p1     0668FF303435554157122853     Pizza #1 at Edwin's Workshop
-p3     0668FF303435554157105440     Pizza #3 at Martin's farm
-```
-
-For the first column, make a unique label, hopefully similar to whatever designation has been agreed to by the team.
-The second column is the serial number obtained by the magic above, and lastly you can have some nice description.
-
-**PLEASE COMMIT CHANGES** after adding your Nucleo to the list. This will help keep things consistently documented as
-we test software on various prototypes.

software/utils/debug/debug.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright 2020-2022, RespiraWorks
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Fail if any command fails
+set -e
+set -o pipefail
+
+# Print each command as it executes
+if [ -n "$VERBOSE" ]; then
+  set -o xtrace
+fi
+
+pushd () {
+    command pushd "$@" > /dev/null
+}
+
+popd () {
+    command popd > /dev/null
+}
+
+# This script should work no matter where you call it from.
+MY_PATH="$(dirname "$0")"
+
+pushd $MY_PATH
+PYTHONPATH=$MY_PATH ./debug_cli.py "${@:1}"
+popd

software/utils/debug/debug_cli.py

@@ -29,15 +29,15 @@
 import os
 import shlex
 import traceback
-from util.colors import *
-from util.error import Error
-from util.serial_detect import DeviceScanner
-from controller_debug import ControllerDebugInterface, MODE_BOOT
-from var_info import VAR_ACCESS_READ_ONLY, VAR_ACCESS_WRITE, VAR_FLOAT_ARRAY
 import numpy as np
 import matplotlib.pyplot as plt
-import test_data
 from pathlib import Path
+from util.colors import *
+from util.error import Error
+from util.serial_detect import DeviceScanner
+from debug_lib.controller_debug import ControllerDebugInterface, MODE_BOOT
+from debug_lib.var_info import VAR_ACCESS_READ_ONLY, VAR_ACCESS_WRITE, VAR_FLOAT_ARRAY
+import debug_lib.test_data
 import debug_funcs
 
 
@@ -82,7 +82,7 @@ def __init__(self, connect_to):
         super(CmdLine, self).__init__()
         self.scripts_directory = "scripts"
         self.test_scenarios_dir = Path("test_scenarios").absolute().resolve()
-        self.test_data_dir = Path("../../../test_data").absolute().resolve()
+        self.test_data_dir = Path("../../../local_data/test_data").absolute().resolve()
         self.interface = ControllerDebugInterface()
         self.maybe_connect(connect_to)
 
@@ -191,27 +191,27 @@ def maybe_connect(self, alias_or_port):
             print(f"Attempting to connect to: {selected.print()}")
             self.interface.connect(selected.port)
 
-    def do_devices(self, line):
+    def do_device(self, line):
         """This command manages the connection of the debug interface to the ventilator controller
 
         If ventilator USB cable is unplugged, the CLI prompt should update to reflect this on the next loop
         (blank line or failed command). You may also start the debugger without a plugged in device.
         Use one of the following commands to reconnect.
 
-        devices list
+        device list
           searches and lists available STM32 devices on the serial bus
 
-        devices off
+        device off
           disconnect if connected
 
-        devices find <alias> [h/p/c]
+        device find <alias> [h/p/c]
           print information for specific device
           alias - string must match alias as defined in manifest
           h - print HLA serial number only
           p - print port only
           c - print configuration only
 
-        devices connect <device>
+        device connect <device>
           connect to device, where <device> is one of:
           - `auto` find port of and connect to plugged in device, must be only one
           - alias, as definded in device manifest

software/utils/debug/controller_debug.py → software/utils/debug/debug_lib/controller_debug.py

@@ -20,17 +20,17 @@
 
 """
 
+import debug_lib.debug_types
+import debug_lib.var_info
+from debug_lib.test_scenario import TestScenario
+from debug_lib.test_data import *
+from util.error import Error
+from util.colors import red, orange
 import serial
 import threading
 import time
-import debug_types
-import var_info
 import fnmatch
-from util.error import Error
-import test_scenario
-from test_data import *
 from pathlib import Path
-from util.colors import red, orange
 
 # TODO: Import constants from proto instead!
 
@@ -327,11 +327,11 @@ def tests_import(self, file_name):
         if not in_file.is_file():
             raise Error(f"Input file does not exist {file_name}")
         elif in_file.suffix == ".csv":
-            imported_scenarios = test_scenario.TestScenario.from_csv(
+            imported_scenarios = TestScenario.from_csv(
                 in_file, self.variable_metadata.keys()
             )
         elif in_file.suffix == ".json":
-            imported_scenarios = test_scenario.TestScenario.from_json(in_file)
+            imported_scenarios = TestScenario.from_json(in_file)
         else:
             raise Error(f"Unknown file format `{in_file.suffix}`")
         if bool(set(imported_scenarios.keys()) & set(self.scenarios.keys())):
@@ -403,7 +403,7 @@ def test_run(self, name):
         return test
 
     def trace_save(self, scenario_name="manual_trace"):
-        test = TestData(test_scenario.TestScenario())
+        test = TestData(TestScenario())
 
         # get data
         self.save_test_data(test)

software/utils/debug/test_data.py → software/utils/debug/debug_lib/test_data.py

@@ -20,16 +20,16 @@
 
 """
 
+from debug_lib.test_scenario import TestScenario
+from util.colors import red
 import datetime
-from test_scenario import TestScenario
-from typing import Dict, List
-from datetime import datetime
 import matplotlib.pyplot as plt
 import dateutil.parser
 import json
 import platform
 import git
-from util.colors import red
+from typing import Dict, List
+from datetime import datetime
 from pathlib import Path
 
 

software/utils/debug/var_info.py → software/utils/debug/debug_lib/var_info.py

@@ -20,7 +20,7 @@
 
 """
 
-import debug_types
+import debug_lib.debug_types
 from util.error import Error
 
 # TODO: Import constants from proto instead!