diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..455da6d --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,7 @@ +# These owners will be the default owners for everything in +# the repo. Unless a later match takes precedence, +# @global-owner1 and @global-owner2 will be requested for +# review when someone opens a pull request. +* @NebraLtd/developers + +# See example for more details: https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/creating-a-repository-on-github/about-code-owners#example-of-a-codeowners-file diff --git a/.github/contributing.md b/.github/contributing.md new file mode 100644 index 0000000..3560acf --- /dev/null +++ b/.github/contributing.md @@ -0,0 +1,35 @@ +# How to contriubte + +Thanks for your interest in contributing to Nebra. We enforce certain rules on commits with the following goals in mind: + +- Be able to reliably auto-generate the `CHANGELOG.md` *without* any human intervention. +- Be able to automatically and correctly increment the semver version number based on what was done since the last release. +- Be able to get a quick overview of what happened to the project by glancing over the commit history. +- Be able to automatically reference relevant changes from a dependency upgrade. + +Our CI will run checks to ensure this guidelines are followed and won't allow merging contributions that don't adhere to them. Version number and changelog are automatically handled by the CI build flow after a pull request is merged. You only need to worry about the commit itself. + +## Commit structure + +Each commit message should consist of a header a body and a footer, structured in the following format: + +``` +: +--BLANK LINE-- +(optional) +--BLANK LINE-- +(optional) Connects-to: #issue-number +(optional) Closes: #issue-number +(mandatory) Change-type: major | minor | patch +(optional) Signed-off-by: Foo Bar +``` + +Note that: +- Blank lines are required to separate header from body and body from footer. You don't need to add two blank lines if you don't add a body. +- `scope`: If your commit touches a well defined component/part/service please addthe scope tag to clarify. Some examples: `docs`, `images`, `typos`. +- `subject`: The subject should contain a short description of the change. Use the imperative, present tense. +- `body`: A detailed description of changes being made and reasoning if necessary. This may contain several paragraphs. +- `Connects-to`: If your commit is connected to an existing issue, link it by adding this tag with `#issue-number`. Example: `Connects-to: #123` +- `Closes`: If your commit fixes an existing issue, link it by adding this tag with `#issue-number`. Example: `Closes: #123` +- `Change-type`: At least one of your commits on a PR needs to have this tag. You have the flexibility, and it's good practise, to use this tag in as many commits as you see fit; in the end, the resulting change type for the scope of the PR will be folded down to the biggest one as marked in the commits (`major>minor>patch`). Our version numbering adheres to [Semantic Versioning](http://semver.org/). +- `Signed-off-by`: Sign your commits by providing your full name and email address in the format: `Name Surname `. *This is an optional tag.* diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..f9195bc --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,9 @@ +**Why** + + +**How** + + + +**References** + diff --git a/.github/workflows/publish-gateway-mfr-rs.yml.bak b/.github/workflows/publish-gateway-mfr-rs.yml.bak new file mode 100644 index 0000000..4e780a3 --- /dev/null +++ b/.github/workflows/publish-gateway-mfr-rs.yml.bak @@ -0,0 +1,72 @@ +name: Periodically check for updates, build and release gateway-mfr-rs + +on: + schedule: + - cron: "0 0 * * 0" # Run weekly on sunday at 00:00 + workflow_dispatch: + +jobs: + rust-compile: + name: Build gateway-mfr-rs + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v2 + - name: Get Latest Release + id: latest_version + uses: abatilo/release-info-action@v1.3.0 + with: + owner: helium + repo: gateway-mfr-rs + - name: Perform check and update + env: + LATEST_GA: ${{ steps.latest_version.outputs.latest_tag }} + run: | + GITHUB_BRANCH=$( echo "${{ github.ref }}" | sed 's/refs\/heads\///g' ) + + echo "LATEST_GA=$LATEST_GA" >> $GITHUB_ENV + echo "GITHUB_BRANCH=$GITHUB_BRANCH" >> $GITHUB_ENV + + # Get the latest GA release + if grep -q "$LATEST_GA" Dockerfile; then + echo "We're on the latest Helium gateway-rs release $LATEST_GA." + exit 0 + else + echo "We're not on the latest Helium gateway-rs release. Updating to $LATEST_GA." + sed -i -E '2 s/GATEWAY_RS_RELEASE=.*/GATEWAY_RS_RELEASE='$LATEST_GA'/g' Dockerfile + UPDATED=true + echo "UPDATED=$UPDATED" >> $GITHUB_ENV + exit 0 + fi + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + target: aarch64-unknown-linux-musl + override: true + + - name: Clone repo + env: + LATEST_GA: ${{ steps.latest_version.outputs.latest_tag }} + run: | + git clone --branch $LATEST_GA https://github.com/helium/gateway-mfr-rs.git . + - name: Run cargo check + uses: actions-rs/cargo@v1 + with: + use-cross: true + command: check + + - name: Build gateway-mfr-rs + uses: actions-rs/cargo@v1 + with: + use-cross: true + command: build + args: --target aarch64-unknown-linux-musl --release + + - name: Copy release file + run: | + cp ./target/aarch64-unknown-linux-musl/release/gateway_mfr gateway_mfr + - uses: actions/upload-artifact@v2 + with: + name: gateway_mfr + path: ./gateway_mfr diff --git a/.github/workflows/publish-to-pypi-test.yml b/.github/workflows/publish-to-pypi-test.yml index 9963ef8..989bae1 100644 --- a/.github/workflows/publish-to-pypi-test.yml +++ b/.github/workflows/publish-to-pypi-test.yml @@ -14,12 +14,12 @@ jobs: uses: actions-rs/toolchain@v1 with: toolchain: stable - target: aarch64-unknown-linux-musl + target: arm-unknown-linux-gnueabihf override: true - name: Clone repo run: | - git clone --branch v0.1.3 https://github.com/helium/gateway-mfr-rs.git . + git clone --branch v0.1.5 https://github.com/helium/gateway-mfr-rs.git . - name: Run cargo check uses: actions-rs/cargo@v1 @@ -32,11 +32,11 @@ jobs: with: use-cross: true command: build - args: --target aarch64-unknown-linux-musl --release + args: --target arm-unknown-linux-gnueabihf --release - name: Copy release file run: | - cp ./target/aarch64-unknown-linux-musl/release/gateway_mfr gateway_mfr + cp ./target/arm-unknown-linux-gnueabihf/release/gateway_mfr gateway_mfr - uses: actions/upload-artifact@v2 with: diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 4fe7fa9..a674e06 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -14,12 +14,12 @@ jobs: uses: actions-rs/toolchain@v1 with: toolchain: stable - target: aarch64-unknown-linux-musl + target: arm-unknown-linux-gnueabihf override: true - name: Clone repo run: | - git clone --branch v0.1.3 https://github.com/helium/gateway-mfr-rs.git . + git clone --branch v0.1.5 https://github.com/helium/gateway-mfr-rs.git . - name: Run cargo check uses: actions-rs/cargo@v1 @@ -32,11 +32,11 @@ jobs: with: use-cross: true command: build - args: --target aarch64-unknown-linux-musl --release + args: --target arm-unknown-linux-gnueabihf --release - name: Copy release file run: | - cp ./target/aarch64-unknown-linux-musl/release/gateway_mfr gateway_mfr + cp ./target/arm-unknown-linux-gnueabihf/release/gateway_mfr gateway_mfr - uses: actions/upload-artifact@v2 with: @@ -81,6 +81,22 @@ jobs: with: name: wheels path: ./dist/* + + - name: Get Latest Release + id: latest_version + uses: abatilo/release-info-action@v1.3.0 + with: + owner: NebraLtd + repo: hm-pyhelper + + - name: Upload binaries to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: ./dist/* + tag: ${{ steps.latest_version.outputs.latest_tag }} + overwrite: true + file_glob: true - name: Publish distribution 📦 to PyPI if: startsWith(github.ref, 'refs/tags') diff --git a/README.md b/README.md index f7da3fe..c916af3 100644 --- a/README.md +++ b/README.md @@ -63,27 +63,29 @@ Please note, DIY Hotspots do not earn HNT. | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | Pi Supply IoT LoRa Gateway HAT | RPi | DIY-PISLGH | 0.0 | 22 | | | Light | False | Any pi with 40 pin header | | RAK2287 | RPi | DIY-RAK2287 | 0.0 | 17 | | | Light | False | Any pi with 40 pin header | -## utils -### logger +## logger ```python -from hm_pyhelper.utils import logger -logger = get_logger(__name__) -logger.debug("message to log") +from hm_pyhelper.logger import get_logger +LOGGER = get_logger(__name__) +LOGGER.debug("message to log") ``` ## miner_param -### get_region -Return the region from envvar REGION_OVERRIDE or -from the contents of /var/pktfwd/region +### retry_get_region(region_override, region_filepath) +Return the region from envvar region_override or +from the contents of region_filepath ```python -from hm_pyhelper.miner_param import get_region -print(get_region()) +from hm_pyhelper.miner_param import retry_get_region +print(retry_get_region("US915", "/invalid/path")) +# US915 -> US9155 +# echo "EU868" > /var/pktfwd/region +print(retry_get_region("", "/var/pktfwd/region")) +# EU868 ``` ## Testing @@ -95,3 +97,34 @@ pip install -r requirements.txt pip install -r test-requirements.txt PYTHONPATH=./ pytest ``` + +## Referencing a branch for development +It is sometimes convenient to use recent changes in hm-pyhelper before an official release. +To do so, first double check that you've added any relevant dependencies to +the `install_requires` section of `setup.py`. Then add the following lines to the +project's Dockerfile. + +```Dockerfile +RUN pip3 install setuptools wheel +RUN pip3 install --target="$OUTPUTS_DIR" git+https://github.com/NebraLtd/hm-pyhelper@BRANCH_NAME +`````` + +## Releasing + +To release, use the [Github new release flow](https://github.com/NebraLtd/hm-pyhelper/releases/new). + +1. Create a new tag in format `vX.Y.Z`. You can use a previously tagged commit, but this is not necessary. +2. Make sure the tag you created matches the value in setup.py. +3. Select `master` as the target branch. If you do not select the master branch, the tag should be in format `vX.Y.Z-rc.N`. +4. Title: `Release vX.Y.Z`. +5. Body: + +**Note: you can create the release notes automatically by selecting the "Auto-generate release notes" option on the releases page.** + +``` +## What's Changed +* Foo +* Bar + +**Full Changelog**: https://github.com/NebraLtd/hm-pyhelper/compare/v0.0.A...v0.0.Z +``` diff --git a/hm_pyhelper/exceptions.py b/hm_pyhelper/exceptions.py new file mode 100644 index 0000000..5c2420d --- /dev/null +++ b/hm_pyhelper/exceptions.py @@ -0,0 +1,6 @@ +class MalformedRegionException(Exception): + pass + + +class SPIUnavailableException(Exception): + pass diff --git a/hm_pyhelper/utils/logger.py b/hm_pyhelper/logger.py similarity index 100% rename from hm_pyhelper/utils/logger.py rename to hm_pyhelper/logger.py diff --git a/hm_pyhelper/miner_param.py b/hm_pyhelper/miner_param.py index 7ca66aa..91cf9c1 100644 --- a/hm_pyhelper/miner_param.py +++ b/hm_pyhelper/miner_param.py @@ -3,9 +3,14 @@ import logging import json from retry import retry -from hm_pyhelper.utils.logger import get_logger +from hm_pyhelper.logger import get_logger +from hm_pyhelper.exceptions import MalformedRegionException, \ + SPIUnavailableException -logger = get_logger(__name__) +LOGGER = get_logger(__name__) +REGION_INVALID_SLEEP_SECONDS = 30 +REGION_FILE_MISSING_SLEEP_SECONDS = 60 +SPI_UNAVAILABLE_SLEEP_SECONDS = 60 def log_stdout_stderr(sp_result): @@ -179,34 +184,37 @@ def get_mac_address(path): return file.readline().strip().upper() -REGION_OVERRIDE_KEY = 'REGION_OVERRIDE' -REGION_FILEPATH = '/var/pktfwd/region' -REGION_INVALID_SLEEP_SECONDS = 30 -REGION_FILE_MISSING_SLEEP_SECONDS = 60 - - -class MalformedRegionException(Exception): - pass - - -@retry(MalformedRegionException, delay=REGION_INVALID_SLEEP_SECONDS, logger=logger) # noqa -@retry(FileNotFoundError, delay=REGION_FILE_MISSING_SLEEP_SECONDS, logger=logger) # noqa -def get_region(): +@retry(MalformedRegionException, delay=REGION_INVALID_SLEEP_SECONDS, logger=LOGGER) # noqa +@retry(FileNotFoundError, delay=REGION_FILE_MISSING_SLEEP_SECONDS, logger=LOGGER) # noqa +def retry_get_region(region_override, region_filepath): """ - Return the region from the environment or parse file created by hm-miner. + Return the override if it exists, or parse file created by hm-miner. + region_override is the actual value, + not the name of the environment variable. Retry if region in file is malformed or not found. """ - region = os.getenv(REGION_OVERRIDE_KEY, False) - if region: - return region + if region_override: + return region_override - logger.debug("No REGION_OVERRIDE defined, will retrieve from miner.") - with open(REGION_FILEPATH) as region_file: + LOGGER.debug("No region override set (value = %s), will retrieve from miner." % region_override) # noqa: E501 + with open(region_filepath) as region_file: region = region_file.read().rstrip('\n') - logger.debug("Region %s parsed from %s " % (region, REGION_FILEPATH)) + LOGGER.debug("Region %s parsed from %s " % (region, region_filepath)) is_region_valid = len(region) > 3 if is_region_valid: return region raise MalformedRegionException("Region %s is invalid" % region) + + +@retry(SPIUnavailableException, delay=SPI_UNAVAILABLE_SLEEP_SECONDS, logger=LOGGER) # noqa +def await_spi_available(spi_bus): + """ + Check that the SPI bus path exists, assuming it is in /dev/{spi_bus} + """ + if os.path.exists('/dev/{}'.format(spi_bus)): + LOGGER.debug("SPI bus %s Configured Correctly" % spi_bus) + return True + else: + raise SPIUnavailableException("SPI bus %s not found!" % spi_bus) diff --git a/hm_pyhelper/tests/test_get_region.py b/hm_pyhelper/tests/test_get_region.py deleted file mode 100644 index 5d50776..0000000 --- a/hm_pyhelper/tests/test_get_region.py +++ /dev/null @@ -1,15 +0,0 @@ -from hm_pyhelper.miner_param import get_region, REGION_OVERRIDE_KEY -import unittest -from unittest.mock import mock_open, patch -import os - - -class TestGetRegion(unittest.TestCase): - def test_get_region_from_override(self): - os.environ[REGION_OVERRIDE_KEY] = 'foo' - self.assertEqual(get_region(), 'foo') - - @patch("builtins.open", new_callable=mock_open, read_data="ZZ111\n") - def test_get_region_from_miner(self, _): - os.environ[REGION_OVERRIDE_KEY] = '' - self.assertEqual(get_region(), 'ZZ111') diff --git a/hm_pyhelper/tests/test_logger.py b/hm_pyhelper/tests/test_logger.py new file mode 100644 index 0000000..86acb27 --- /dev/null +++ b/hm_pyhelper/tests/test_logger.py @@ -0,0 +1,29 @@ +from unittest import TestCase +from hm_pyhelper.logger import get_logger, _log_format +import re +import logging + + +class TestLogger(TestCase): + def test_get_logger(self): + logger = get_logger(__name__) + + with self.assertLogs() as captured: + logger.debug("Hello world.") + + # check that there is only one log message + self.assertEqual(len(captured.records), 1) + record = captured.records[0] + formatter = logging.Formatter(_log_format) + formatted_output = formatter.format(record) + + # Do not check timestamp and filepath because those change + # based on the environment and run time + expected_partial_output_regex = re.escape( + " - [DEBUG] - hm_pyhelper.tests.test_logger -" + + " (test_logger.py).test_get_logger -- ") + expected_output_regex = ".*" + \ + expected_partial_output_regex + ".*" + \ + " - Hello world." + are_logs_correct = re.search(expected_output_regex, formatted_output) + self.assertTrue(are_logs_correct) diff --git a/hm_pyhelper/tests/test_miner_param.py b/hm_pyhelper/tests/test_miner_param.py index 2a4480f..0cfe9ff 100644 --- a/hm_pyhelper/tests/test_miner_param.py +++ b/hm_pyhelper/tests/test_miner_param.py @@ -1,8 +1,7 @@ +from hm_pyhelper.miner_param import retry_get_region, await_spi_available, \ + provision_key, did_gateway_mfr_test_result_include_miner_key_pass import unittest -from unittest.mock import patch -from hm_pyhelper.miner_param import provision_key -from hm_pyhelper.miner_param import \ - did_gateway_mfr_test_result_include_miner_key_pass +from unittest.mock import mock_open, patch ALL_PASS_GATEWAY_MFR_TESTS = [ { @@ -77,7 +76,6 @@ def stdout(): class TestMinerParam(unittest.TestCase): - @patch( 'hm_pyhelper.miner_param.get_gateway_mfr_test_result', return_value={ @@ -161,3 +159,14 @@ def test_did_gateway_mfr_test_result_include_miner_key_pass(self): get_gateway_mfr_test_result ) ) + + def test_get_region_from_override(self): + self.assertEqual(retry_get_region("foo", "bar/"), 'foo') + + @patch("builtins.open", new_callable=mock_open, read_data="ZZ111\n") + def test_get_region_from_miner(self, _): + self.assertEqual(retry_get_region(False, "foo/"), 'ZZ111') # noqa: E501 + + @patch("os.path.exists", return_value=True) + def test_is_spi_available(self, _): + self.assertTrue(await_spi_available("spiXY.Z")) diff --git a/hm_pyhelper/tests/utils/test_logger.py b/hm_pyhelper/tests/utils/test_logger.py deleted file mode 100644 index d2111ad..0000000 --- a/hm_pyhelper/tests/utils/test_logger.py +++ /dev/null @@ -1,46 +0,0 @@ -from unittest import TestCase -from hm_pyhelper.utils.logger import get_logger, _log_format -import re -import logging - - -class TestExample(TestCase): - def test_logging(self): - logger = get_logger(__name__) - - with self.assertLogs() as captured: - logger.debug("Hello world.") - - # check that there is only one log message - self.assertEqual(len(captured.records), 1) - record = captured.records[0] - formatter = logging.Formatter(_log_format) - formatted_output = formatter.format(record) - - # Do not check timestamp and filepath because those change - # based on the environment and run time - expected_partial_output_regex = re.escape( - " - [DEBUG] - test_logger - (test_logger.py).test_logging -- ") - expected_output_regex = ".*" + \ - expected_partial_output_regex + ".*" + \ - " - Hello world." - are_logs_correct = re.search(expected_output_regex, formatted_output) - self.assertTrue(are_logs_correct) - - def test_custom_log_format(self): - log_string = f"[%(levelname)s] - %(name)s: %(message)s" # noqa: F541 E501 - log_message = "test log string" - logger = get_logger(__name__, log_string) - - with self.assertLogs() as captured: - logger.info(log_message) - - # check that there is only one log message - self.assertEqual(len(captured.records), 1) - record = captured.records[0] - formatter = logging.Formatter(log_string) - formatted_output = formatter.format(record) - - result = re.match("\\[INFO\\]\\s-\\s.*:\\s" + log_message, - formatted_output) - self.assertTrue(result) diff --git a/setup.py b/setup.py index dc82b34..cb6f612 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,8 @@ url="https://github.com/NebraLtd/hm-pyhelper", install_requires=[ 'requests>=2.26.0', - 'jsonrpcclient==3.3.6' + 'jsonrpcclient==3.3.6', + 'retry==0.9.2' ], project_urls={ "Bug Tracker": "https://github.com/NebraLtd/hm-pyhelper/issues",