feat: allow override of key location and remove old version support (#…

…227)

* feat: allow override of key location and remove old version support

This is a quick and hacky fix to the issue described here NebraLtd/helium-syncrobit#2 (comment)

The idea is to allow a device level environment variable `SWARM_KEY_URI_OVERRIDE` in balenaCloud to override the key location in the assigned variant without needing to define a new variant. This is useful for testing new devices or in instances like this Syncrobit one to provide a quick fix until we have implemented a "proper" fix to this issue.

This additionally changes the following:
- we hard code the gateway_mfr_rs version in the [test](https://github.com/NebraLtd/hm-pyhelper/blob/master/.github/workflows/publish-to-pypi-test.yml) and [production](https://github.com/NebraLtd/hm-pyhelper/blob/master/.github/workflows/publish-to-pypi.yml) python package build actions, so there is no need to support old versions of gateway-mfr-rs
- due to removing the above, there is also no need to keep the KEY_STORAGE_BUS key in the hardware variants as this is not used anywhere else that I'm aware of **(we should double check this in config, diag, miner, packet forwarder and multiplexer containers just to be 100% sure)**
  - [Used as a fallback in diag](https://github.com/NebraLtd/hm-diag/blob/23c66980f75568a4cecb18c5375a08cd67bff550/hw_diag/utilities/hardware.py#L200) but not required

Relates-to: NebraLtd/helium-syncrobit#2
Relates-to: #221
Relates-to: #222

hm_pyhelper/hardware_definitions.py

@@ -26,7 +26,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'arm64',
         'BALENA_DEVICE_TYPE': ['raspberrypi3-64'],
         'SPIBUS': 'spidev1.2',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 38,
         'MAC': 'eth0',
@@ -48,7 +47,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'arm64',
         'BALENA_DEVICE_TYPE': ['raspberrypi3-64'],
         'SPIBUS': 'spidev1.2',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 38,
         'MAC': 'eth0',
@@ -65,14 +63,13 @@ def is_raspberry_pi() -> bool:
         },
 
     # Nebra Outdoor Hotspot Gen2
-    # Note:: assuming outdoor2 is equivalent to light miner 2
+    # Note:: assuming outdoor2 is equivalent to indoor2
     'nebra-outdoor2': {
         'FRIENDLY': 'Nebra ROCK Pi 4 Outdoor',
         'SUPPORTED_MODELS': ['Nebra ROCK Pi 4 Outdoor'],
         'CPU_ARCH': 'arm64',
         'BALENA_DEVICE_TYPE': ['rockpi-4b-rk3399'],
         'SPIBUS': 'spidev32766.0',
-        'KEY_STORAGE_BUS': '/dev/i2c-7',
         'SWARM_KEY_URI': 'ecc://i2c-7:96?slot=0',
         'RESET': 149,
         'MAC': 'eth0',
@@ -98,7 +95,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'armv6hf',
         'BALENA_DEVICE_TYPE': ['raspberry-pi'],
         'SPIBUS': 'spidev1.2',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 22,
         'MAC': 'wlan0',
@@ -120,7 +116,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'armv6hf',
         'BALENA_DEVICE_TYPE': ['raspberry-pi'],
         'SPIBUS': 'spidev1.2',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 22,
         'MAC': 'wlan0',
@@ -142,7 +137,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'arm64',
         'BALENA_DEVICE_TYPE': ['radxa-zero'],
         'SPIBUS': 'spidev0.0',
-        'KEY_STORAGE_BUS': '/dev/i2c-3',
         'SWARM_KEY_URI': 'ecc://i2c-3:96?slot=0',
         'RESET': 415,
         'MAC': 'wlan0',
@@ -164,7 +158,7 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'armv6hf',
         'BALENA_DEVICE_TYPE': ['raspberry-pi'],
         'SPIBUS': 'spidev1.2',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
+        'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 4,
         'MAC': 'wlan0',
         'STATUS': 26,
@@ -185,7 +179,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'armv7hf',
         'BALENA_DEVICE_TYPE': ['beaglebone-black'],
         'SPIBUS': 'spidev1.0',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 60,
         'MAC': 'eth0',
@@ -207,7 +200,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'armv7hf',
         'BALENA_DEVICE_TYPE': ['beaglebone-pocket'],
         'SPIBUS': 'spidev1.2',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 60,
         'MAC': 'wlan0',
@@ -229,7 +221,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'arm64',
         'BALENA_DEVICE_TYPE': ['rockpi-4b-rk3399'],
         'SPIBUS': 'spidev32766.0',
-        'KEY_STORAGE_BUS': '/dev/i2c-7',
         'SWARM_KEY_URI': 'ecc://i2c-7:96?slot=0',
         'RESET': 149,
         'MAC': 'eth0',
@@ -253,7 +244,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'armv7hf',
         'BALENA_DEVICE_TYPE': ['asus-tinker-board'],
         'SPIBUS': 'spidev2.0',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 167,
         'MAC': 'eth0',
@@ -275,7 +265,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'arm64',
         'BALENA_DEVICE_TYPE': ['raspberrypi4-64'],
         'SPIBUS': 'spidev0.0',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 25,
         'MAC': 'wlan0',
@@ -297,8 +286,8 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'arm64',
         'BALENA_DEVICE_TYPE': ['raspberrypi3-64', 'raspberrypi4-64'],
         'SPIBUS': 'spidev0.0',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
+        'ONBOARDING_KEY_URI': 'ecc://i2c-1:96?slot=15',
         'RESET': 25,
         'MAC': 'wlan0',
         'STATUS': 20,
@@ -319,7 +308,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'arm64',
         'BALENA_DEVICE_TYPE': ['raspberrypi4-64'],
         'SPIBUS': 'spidev0.0',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 17,
         'MAC': 'wlan0',
@@ -341,7 +329,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'arm64',
         'BALENA_DEVICE_TYPE': ['raspberrypi4-64'],
         'SPIBUS': 'spidev0.0',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 23,
         'MAC': 'wlan0',
@@ -363,7 +350,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'arm64',
         'BALENA_DEVICE_TYPE': ['raspberrypi4-64'],
         'SPIBUS': 'spidev0.0',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 17,
         'MAC': 'wlan0',
@@ -390,7 +376,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'arm64',
         'BALENA_DEVICE_TYPE': ['raspberrypi4-64'],
         'SPIBUS': 'spidev0.0',
-        'KEY_STORAGE_BUS': '/dev/i2c-0',
         'SWARM_KEY_URI': 'ecc://i2c-0:96?slot=0',
         'RESET': 23,
         'MAC': 'eth0',
@@ -412,7 +397,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'arm64',
         'BALENA_DEVICE_TYPE': ['raspberrypi4-64'],
         'SPIBUS': 'spidev0.0',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 4,
         'MAC': 'wlan0',
@@ -441,7 +425,6 @@ def is_raspberry_pi() -> bool:
                                'raspberrypi0-2w-64', 'raspberrypi3-64',
                                'raspberrypi4-64'],
         'SPIBUS': 'spidev0.0',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 22,
         'MAC': 'eth0',
@@ -465,7 +448,6 @@ def is_raspberry_pi() -> bool:
                                'raspberrypi0-2w-64', 'raspberrypi3-64',
                                'raspberrypi4-64'],
         'SPIBUS': 'spidev0.0',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 17,
         'MAC': 'eth0',
@@ -487,7 +469,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'arm64',
         'BALENA_DEVICE_TYPE': ['raspberrypicm4-ioboard'],
         'SPIBUS': 'spidev0.0',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 17,
         'MAC': 'wlan0',
@@ -509,7 +490,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'arm64',
         'BALENA_DEVICE_TYPE': ['raspberrypicm4-ioboard'],
         'SPIBUS': 'spidev0.0',
-        'KEY_STORAGE_BUS': '/dev/i2c-10',
         'SWARM_KEY_URI': 'ecc://i2c-10:96?slot=0',
         'RESET': 23,
         'MAC': 'wlan0',
@@ -531,7 +511,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'arm64',
         'BALENA_DEVICE_TYPE': ['raspberrypicm4-ioboard'],
         'SPIBUS': 'spidev0.0',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 17,
         'MAC': 'wlan0',
@@ -553,7 +532,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'arm64',
         'BALENA_DEVICE_TYPE': ['raspberrypi4-64'],
         'SPIBUS': 'spidev0.0',  # There is a CSN1 pin which is connected to GPIO6 (HAT Pin 31)
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 22,
         'MAC': 'eth0',
@@ -575,7 +553,6 @@ def is_raspberry_pi() -> bool:
         'CPU_ARCH': 'arm64',
         'BALENA_DEVICE_TYPE': ['raspberrypicm4-ioboard'],
         'SPIBUS': 'spidev0.0',
-        'KEY_STORAGE_BUS': '/dev/i2c-1',
         'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0',
         'RESET': 17,
         'MAC': 'wlan0',

hm_pyhelper/miner_param.py

@@ -1,4 +1,5 @@
 import os
+import re
 import subprocess
 import json
 import platform
@@ -26,7 +27,7 @@
 
 
 @lock_ecc()
-def run_gateway_mfr(sub_command: str, slot: int = 0) -> dict:
+def run_gateway_mfr(sub_command: str, slot: int = False) -> dict:
     command = get_gateway_mfr_command(sub_command, slot=slot)
 
     try:
@@ -110,39 +111,31 @@ def get_gateway_mfr_version() -> Version:
         raise GatewayMFRInvalidVersion(err_str).with_traceback(e.__traceback__)
 
 
-def get_gateway_mfr_command(sub_command: str, slot: int = 0) -> list:
+def get_gateway_mfr_command(sub_command: str, slot: int = False) -> list:
     gateway_mfr_path = get_gateway_mfr_path()
     command = [gateway_mfr_path]
 
     gateway_mfr_version = get_gateway_mfr_version()
-    if Version('0.1.1') < gateway_mfr_version < Version('0.2.0'):
-        try:
-            device_arg = [
-                '--path',
-                get_variant_attribute(os.getenv('VARIANT'), 'KEY_STORAGE_BUS')
-            ]
-            command.extend(device_arg)
-        except (UnknownVariantException, UnknownVariantAttributeException) as e:
-            LOGGER.warning(str(e) + ' Omitting --path arg.')
-
-        command.append(sub_command)
 
-        # In case of "key" command, append the slot number 0 at the end.
-        if sub_command == "key":
-            command.append("0")
-
-    elif gateway_mfr_version >= Version('0.2.0'):
+    if gateway_mfr_version >= Version('0.2.0'):
         try:
+            if os.getenv('SWARM_KEY_URI_OVERRIDE'):
+                ecc_location = os.getenv('SWARM_KEY_URI_OVERRIDE')
+            else:
+                ecc_location = get_variant_attribute(os.getenv('VARIANT'), 'SWARM_KEY_URI')
+
             device_arg = [
                 '--device',
-                get_variant_attribute(os.getenv('VARIANT'), 'SWARM_KEY_URI')
+                ecc_location
             ]
             command.extend(device_arg)
         except (UnknownVariantException, UnknownVariantAttributeException) as e:
             LOGGER.warning(str(e) + ' Omitting --device arg.')
 
-        slot_str = f'slot={slot}'
-        command[-1] = command[-1].replace('slot=0', slot_str)
+        if slot:
+            slot_str = f'slot={slot}'
+            slot_pattern = r'(slot=\d+)'
+            command[-1] = re.sub(slot_pattern, slot_str, command[-1])
 
         if ' ' in sub_command:
             command += sub_command.split(' ')

hm_pyhelper/tests/test_miner_param.py

@@ -8,7 +8,7 @@
     GatewayMFRFileNotFoundException, UnsupportedGatewayMfrVersion
 from hm_pyhelper.lock_singleton import ResourceBusyError
 from hm_pyhelper.miner_param import retry_get_region, await_spi_available, \
-    provision_key, run_gateway_mfr, \
+    provision_key, run_gateway_mfr, get_gateway_mfr_path, \
     did_gateway_mfr_test_result_include_miner_key_pass, \
     get_mac_address, get_public_keys_rust, get_gateway_mfr_version, get_gateway_mfr_command
 
@@ -119,44 +119,6 @@ def test_get_gateway_mfr_version_v021(self, mocked_subprocess_run):
         mocked_subprocess_run.assert_called_once_with(
             [ANY, '--version'], capture_output=True, check=True)
 
-    @patch('hm_pyhelper.miner_param.get_gateway_mfr_version',
-           return_value=Version('0.1.7'))
-    def test_get_gateway_mfr_command_v017(self, mocked_get_gateway_mfr_version):
-        actual_result = get_gateway_mfr_command('key')
-        expected_result = [ANY, '--path', '/dev/i2c-X', 'key', '0']
-        self.assertListEqual(actual_result, expected_result)
-        mocked_get_gateway_mfr_version.assert_called_once()
-
-        actual_result = get_gateway_mfr_command('info')
-        expected_result = [ANY, '--path', '/dev/i2c-X', 'info']
-        self.assertListEqual(actual_result, expected_result)
-
-    @patch.dict('os.environ', {"VARIANT": "NEBHNT-INVALID"})
-    @patch('hm_pyhelper.miner_param.get_gateway_mfr_version',
-           return_value=Version('0.1.7'))
-    def test_get_gateway_mfr_command_v017_no_variant(self, mocked_get_gateway_mfr_version):
-        actual_result = get_gateway_mfr_command('key')
-        expected_result = [ANY, 'key', '0']
-        self.assertListEqual(actual_result, expected_result)
-        mocked_get_gateway_mfr_version.assert_called_once()
-
-        actual_result = get_gateway_mfr_command('info')
-        expected_result = [ANY, 'info']
-        self.assertListEqual(actual_result, expected_result)
-
-    @patch.dict('os.environ', {"VARIANT": "NEBHNT-NO-ECC-ADDRESS"})
-    @patch('hm_pyhelper.miner_param.get_gateway_mfr_version',
-           return_value=Version('0.1.7'))
-    def test_get_gateway_mfr_command_v017_no_KEY_STORAGE_BUS(self, mocked_get_gateway_mfr_version):
-        actual_result = get_gateway_mfr_command('key')
-        expected_result = [ANY, 'key', '0']
-        self.assertListEqual(actual_result, expected_result)
-        mocked_get_gateway_mfr_version.assert_called_once()
-
-        actual_result = get_gateway_mfr_command('info')
-        expected_result = [ANY, 'info']
-        self.assertListEqual(actual_result, expected_result)
-
     @patch('hm_pyhelper.miner_param.get_gateway_mfr_version',
            return_value=Version('0.2.1'))
     def test_get_gateway_mfr_command_v021(self, mocked_get_gateway_mfr_version):
@@ -165,8 +127,8 @@ def test_get_gateway_mfr_command_v021(self, mocked_get_gateway_mfr_version):
         self.assertListEqual(actual_result, expected_result)
         mocked_get_gateway_mfr_version.assert_called_once()
 
-        actual_result = get_gateway_mfr_command('info')
-        expected_result = [ANY, '--device', 'ecc://i2c-X:96?slot=0', 'info']
+        actual_result = get_gateway_mfr_command('test', 9)
+        expected_result = [ANY, '--device', 'ecc://i2c-X:96?slot=9', 'test']
         self.assertListEqual(actual_result, expected_result)
 
     @patch.dict('os.environ', {"VARIANT": "NEBHNT-INVALID"})
@@ -205,6 +167,7 @@ def test_get_gateway_mfr_command_v020(self, mocked_get_gateway_mfr_version):
 
         actual_result = get_gateway_mfr_command('info')
         expected_result = [ANY, '--device', 'ecc://i2c-X:96?slot=0', 'info']
+        self.assertListEqual(actual_result, expected_result)
 
     @patch('hm_pyhelper.miner_param.get_gateway_mfr_version',
            return_value=Version('0.3.9'))
@@ -216,6 +179,19 @@ def test_get_gateway_mfr_command_v021_upward(self, mocked_get_gateway_mfr_versio
 
         actual_result = get_gateway_mfr_command('info')
         expected_result = [ANY, '--device', 'ecc://i2c-X:96?slot=0', 'info']
+        self.assertListEqual(actual_result, expected_result)
+
+    @patch('hm_pyhelper.miner_param.get_gateway_mfr_version',
+           return_value=Version('0.3.9'))
+    def test_get_gateway_mfr_command_slot_force(self, mocked_get_gateway_mfr_version):
+        actual_result = get_gateway_mfr_command('key', 3)
+        expected_result = [ANY, '--device', 'ecc://i2c-X:96?slot=3', 'key']
+        self.assertListEqual(actual_result, expected_result)
+        mocked_get_gateway_mfr_version.assert_called_once()
+
+        actual_result = get_gateway_mfr_command('info', 15)
+        expected_result = [ANY, '--device', 'ecc://i2c-X:96?slot=15', 'info']
+        self.assertListEqual(actual_result, expected_result)
 
     @patch('hm_pyhelper.miner_param.get_gateway_mfr_version',
            return_value=Version('0.0.0'))
@@ -352,3 +328,12 @@ def test_is_spi_available(self, _):
     def test_error_mac_address(self):
         with pytest.raises(MinerFailedToFetchMacAddress):
             get_mac_address("test/path")
+
+    @patch('platform.machine')
+    @patch('os.path.dirname')
+    def test_get_gateway_mfr_path(self, mock_dir, mock_platform):
+        mock_dir.return_value = "/test/this/works"
+        mock_platform.return_value = "aarch64"
+        actual_result = get_gateway_mfr_path()
+        expected_result = "/test/this/works/gateway_mfr_aarch64"
+        self.assertEqual(actual_result, expected_result)

setup.py

@@ -3,7 +3,7 @@
 
 setup(
     name='hm_pyhelper',
-    version='0.13.54',
+    version='0.13.55',
     author="Nebra Ltd",
     author_email="[email protected]",
     description="Helium Python Helper",