Merge with develop

.github/CODEOWNERS

@@ -1 +1 @@
-*       @akuker @erichelgeson @rdmark
+*       @akuker @rdmark

.github/ISSUE_TEMPLATE.md

@@ -4,6 +4,7 @@
 - Which github revision of software:
 - Which board version:
 - Which computer is the PiSCSI connected to:
+- Which OS you are using (output of 'lsb_release -a'):
 
 
 # Describe the issue

.github/workflows/cpp.yml

@@ -1,20 +1,17 @@
-name: C++ Tests/Analysis
+name: C++ Tests; Full Static Analysis
 
 on:
   workflow_dispatch:
   push:
     paths:
       - 'cpp/**'
+      - 'python/**'
       - '.github/workflows/cpp.yml'
   pull_request:
     paths:
       - 'cpp/**'
+      - 'python/**'
       - '.github/workflows/cpp.yml'
-    types:
-      - assigned
-      - opened
-      - synchronize
-      - reopened
     branches:
       - 'develop'
       - 'main'
@@ -118,4 +115,4 @@ jobs:
           --define sonar.coverage.exclusions="cpp/**/test/**"
           --define sonar.cpd.exclusions="cpp/**/test/**"
           --define sonar.inclusions="cpp/**,python/**"
-          --define sonar.python.version=3.7,3.9
+          --define sonar.python.version=3.9,3.11

.github/workflows/web.yml

@@ -14,11 +14,6 @@ on:
       - 'python/common/**'
       - '.github/workflows/web.yml'
       - 'easyinstall.sh'
-    types:
-      - assigned
-      - opened
-      - synchronize
-      - reopened
     branches:
       - 'develop'
       - 'main'

easyinstall.sh

@@ -76,7 +76,7 @@ SECRET_FILE="$HOME/.config/piscsi/secret"
 FILE_SHARE_PATH="$HOME/shared_files"
 FILE_SHARE_NAME="Pi File Server"
 
-APT_PACKAGES_COMMON="build-essential git protobuf-compiler bridge-utils"
+APT_PACKAGES_COMMON="build-essential git protobuf-compiler bridge-utils ca-certificates"
 APT_PACKAGES_BACKEND="libspdlog-dev libpcap-dev libprotobuf-dev protobuf-compiler libgmock-dev clang"
 APT_PACKAGES_PYTHON="python3 python3-dev python3-pip python3-venv python3-setuptools python3-wheel libev-dev libevdev2"
 APT_PACKAGES_WEB="nginx-light genisoimage man2html hfsutils dosfstools kpartx unzip unar disktype gettext"
@@ -111,7 +111,15 @@ function sudoCheck() {
 function deleteFile() {
     if sudo test -f "$1/$2"; then
         sudo rm "$1/$2" || exit 1
-        echo "Deleted $1/$2"
+        echo "Deleted file $1/$2"
+    fi
+}
+
+# Delete dir if it exists
+function deleteDir() {
+    if sudo test -d "$1"; then
+        sudo rm -rf "$1" || exit 1
+        echo "Deleted directory $1"
     fi
 }
 
@@ -229,6 +237,9 @@ function installPiscsiWebInterface() {
     deleteFile "$SSL_CERTS_PATH" "rascsi-web.crt"
     deleteFile "$SSL_KEYS_PATH" "rascsi-web.key"
 
+    # Deleting previous venv dir, if one exists, to avoid the common issue of broken python dependencies
+    deleteDir "$WEB_INSTALL_PATH/venv"
+
     if [ -f "$SSL_CERTS_PATH/piscsi-web.crt" ]; then
         echo "SSL certificate $SSL_CERTS_PATH/piscsi-web.crt already exists."
     else
@@ -937,6 +948,7 @@ function installSamba() {
 # Installs and configures Webmin
 function installWebmin() {
     WEBMIN_PATH="/usr/share/webmin"
+    WEBMIN_MODULE_CONFIG="/etc/webmin/netatalk2/config"
     WEBMIN_MODULE_VERSION="1.0"
 
     if [ -d "$WEBMIN_PATH" ]; then
@@ -953,17 +965,27 @@ function installWebmin() {
 
     echo
     echo "Installing packages..."
-    sudo apt-get install curl --no-install-recommends --assume-yes </dev/null
+    sudo apt-get install curl libcgi-session-perl --no-install-recommends --assume-yes </dev/null
     curl -o setup-repos.sh https://raw.githubusercontent.com/webmin/webmin/master/setup-repos.sh
     sudo sh setup-repos.sh
     rm setup-repos.sh
-    sudo apt-get install webmin --install-recommends
+    sudo apt-get install webmin --install-recommends </dev/null
     echo
     echo "Downloading and installing Webmin module..."
-    rm netatalk2-wbm.tgz || true
+    if [[ -f "$WEBMIN_MODULE_CONFIG" ]]; then
+        echo "$WEBMIN_MODULE_CONFIG already exists; will not modify..."
+	WEBMIN_MODULE_FLAG=1
+    fi
+
+    rm netatalk2-wbm.tgz 2> /dev/null || true
     wget -O netatalk2-wbm.tgz "https://github.com/Netatalk/netatalk-webmin/releases/download/netatalk2-$WEBMIN_MODULE_VERSION/netatalk2-wbm-$WEBMIN_MODULE_VERSION.tgz" </dev/null
-    sudo /usr/share/webmin/install-module.pl netatalk2-wbm.tgz
-    rm netatalk2-wbm.tgz
+    sudo "$WEBMIN_PATH/install-module.pl" netatalk2-wbm.tgz
+
+    if [[ ! $WEBMIN_MODULE_FLAG ]]; then
+        echo "Modifying $WEBMIN_MODULE_CONFIG..."
+        sudo sed -i 's@/usr/sbin@/usr/local/sbin@' "$WEBMIN_MODULE_CONFIG"
+    fi
+    rm netatalk2-wbm.tgz || true
 }
 
 # updates configuration files and installs packages needed for the OLED screen script
@@ -1025,6 +1047,9 @@ function installPiscsiScreen() {
         REBOOT=1
     fi
 
+    # Deleting previous venv dir, if one exists, to avoid the common issue of broken python dependencies
+    deleteDir "$OLED_INSTALL_PATH/venv"
+
     echo "Installing the piscsi-oled.service configuration..."
     sudo cp -f "$OLED_INSTALL_PATH/service-infra/piscsi-oled.service" "$SYSTEMD_PATH/piscsi-oled.service"
     sudo sed -i /^ExecStart=/d "$SYSTEMD_PATH/piscsi-oled.service"
@@ -1129,6 +1154,9 @@ function installPiscsiCtrlBoard() {
     fi
     set -e
 
+    # Deleting previous venv dir, if one exists, to avoid the common issue of broken python dependencies
+    deleteDir "$CTRLBOARD_INSTALL_PATH/venv"
+
     echo "Installing the piscsi-ctrlboard.service configuration..."
     sudo cp -f "$CTRLBOARD_INSTALL_PATH/service-infra/piscsi-ctrlboard.service" "$SYSTEMD_PATH/piscsi-ctrlboard.service"
     sudo sed -i /^ExecStart=/d "$SYSTEMD_PATH/piscsi-ctrlboard.service"
@@ -1223,7 +1251,7 @@ function runChoice() {
               compilePiscsi
               backupPiscsiService
               installPiscsi
-	      configurePiscsiService
+              configurePiscsiService
               enablePiscsiService
               preparePythonCommon
               if [[ $(isPiscsiScreenInstalled) -eq 0 ]]; then
@@ -1266,7 +1294,7 @@ function runChoice() {
               backupPiscsiService
               preparePythonCommon
               installPiscsi
-	      configurePiscsiService
+              configurePiscsiService
               enablePiscsiService
               if [[ $(isPiscsiScreenInstalled) -eq 0 ]]; then
                   echo "Detected piscsi oled service; will run the installation steps for the OLED monitor."
@@ -1454,7 +1482,7 @@ function runChoice() {
               fetchHardDiskDrivers
               compilePiscsi
               installPiscsi
-	      configurePiscsiService
+              configurePiscsiService
               enablePiscsiService
               preparePythonCommon
               cachePipPackages

python/README.md

@@ -6,18 +6,47 @@ packages that are shared among the clients.
 The following paragraphs in this README contain instructions that are shared 
 among all Python apps.
 
-## Supported Python interpreter
+## Supported Python interpreters
 
 The policy in this project is to support the Python 3 interpreter that comes 
-standard with the current stable, as well as previous stable release of Debian.
+standard with the current stable, as well as the previous stable releases of Debian.
 
 At the time of writing they are:
-- Python 3.9.2 in [Debian Bullseye](https://packages.debian.org/bullseye/python3)
-- Python 3.7.3 in [Debian Buster](https://packages.debian.org/buster/python3)
+- Python 3.11 in [Debian Bookworm](https://packages.debian.org/bookworm/python3)
+- Python 3.9 in [Debian Bullseye](https://packages.debian.org/bullseye/python3)
 
-## Static analysis with pylint
+## Dependencies
 
-It is recommended to run pylint against new code to protect against bugs
+We use 'pip freeze' to manage explicit Python dependencies in this project.
+After adding new or bumping the versions of Python dependencies,
+please run the following command in the requisite subdir commit the results:
+
+```
+pip freeze -l > requirements.txt
+```
+
+## Static analysis and formatting
+
+The CI workflow is set up to check code formatting with `black`,
+and linting with `flake8`. If non-conformant code is found, the CI job
+will fail.
+
+Before checking in new code, install the development packages and run
+these two tools locally.
+
+```
+pip install -r web/requirements-dev.txt
+```
+
+Note that `black` only works correctly if you run it in the root of the
+`python/` dir:
+
+```
+cd python
+black .
+```
+
+Optionally: It is recommended to run pylint against new code to protect against bugs
 and keep the code readable and maintainable.
 The local pylint configuration lives in .pylintrc.
 In order for pylint to recognize venv libraries, the pylint-venv package is required.

python/common/requirements.txt

@@ -1,3 +1,7 @@
+certifi==2023.7.22
+charset-normalizer==3.3.2
+idna==3.4
 protobuf==3.19.5
 requests==2.31.0
+urllib3==2.0.7
 vcgencmd==0.1.1