Add container-based build workflow and improve build tooling

.github/workflows/build.yaml

@@ -127,48 +127,12 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install SDK extensions
-        run: |
-          # XXX: slc-cli does not actually work when the extensions aren't in the SDK!
-          for sdk in /*_sdk_*; do
-            slc signature trust --sdk "$sdk"
-
-            ln -s $PWD/gecko_sdk_extensions "$sdk"/extension
-
-            for ext in "$sdk"/extension/*/; do
-              slc signature trust --sdk "$sdk" --extension-path "$ext"
-            done
-          done
-
       - name: Build firmware
         id: build-firmware
         run: |
-          # Fix `fatal: detected dubious ownership in repository at`
-          git config --global --add safe.directory "$GITHUB_WORKSPACE"
-
-          # Pass all SDKs as consecutive `--sdk ...` arguments
-          sdk_args=""
-          for sdk_dir in /*_sdk*; do
-            sdk_args="$sdk_args --sdk $sdk_dir"
-          done
-
-          # Pass all toolchains as consecutive `--toolchain ...` arguments
-          toolchain_args=""
-          for toolchain_dir in /opt/*arm-none-eabi*; do
-            toolchain_args="$toolchain_args --toolchain $toolchain_dir"
-          done
-
-          # Build it
-          /opt/venv/bin/python3 tools/build_project.py \
-            $sdk_args \
-            $toolchain_args \
-            --manifest "${{ matrix.manifest }}" \
-            --build-dir build \
-            --build-system makefile \
-            --output-dir outputs \
-            --output gbl \
-            --output hex \
-            --output out
+          build_firmware.sh \
+            --work-dir "$GITHUB_WORKSPACE" \
+            --manifest "${{ matrix.manifest }}"
 
           # Get the basename of the GBL in `outputs`
           output_basename=$(basename -- $(basename -- $(ls -1 outputs/*.gbl | head -n 1)) .gbl)

.github/workflows/shellcheck.yml

@@ -0,0 +1,34 @@
+name: reviewdog / shellscripts
+
+on:
+    push:
+        paths:
+            - tools/build_firmware.sh
+
+    pull_request:
+        paths:
+            - tools/build_firmware.sh
+
+jobs:
+
+  shellcheck:
+    name: Check shell scripts with shellcheck
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: shellcheck
+        uses: reviewdog/action-shellcheck@v1
+        with:
+          reporter: github-pr-review
+          path: tools
+          pattern: "*.sh"
+
+  shfmt:
+    name: Check shell scripts with shfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: reviewdog/action-shfmt@v1
+        with:
+          level: warning
+          shfmt_flags: '-i 0 '

.gitignore

@@ -9,6 +9,8 @@ autogen/
 *.project.mak
 *.Makefile
 build/
+outputs/
+build_dir/
 artifact/
 artifacts/
 trashed_modified_files/

Dockerfile

@@ -22,6 +22,7 @@ RUN \
        xz-utils
 
 COPY requirements.txt /tmp/
+COPY tools/build_firmware.sh /usr/bin/build_firmware.sh
 
 RUN \
     virtualenv /opt/venv \
@@ -30,8 +31,10 @@ RUN \
 
 # Install Simplicity Commander (unfortunately no stable URL available, this
 # is known to be working with Commander_linux_x86_64_1v15p0b1306.tar.bz).
+ARG SIMPLICITY_COMMANDER_SHA256SUM=ce7b9138c54f4fa0a24c48c8347e55e3e5f8b402d7f32615771bd0403c2d8962
 RUN \
     curl -O https://www.silabs.com/documents/login/software/SimplicityCommander-Linux.zip \
+    && echo "${SIMPLICITY_COMMANDER_SHA256SUM}  SimplicityCommander-Linux.zip" | sha256sum -c \
     && unzip -q SimplicityCommander-Linux.zip \
     && tar -C /opt -xjf SimplicityCommander-Linux/Commander_linux_x86_64_*.tar.bz \
     && rm -r SimplicityCommander-Linux \
@@ -40,34 +43,44 @@ RUN \
 ENV PATH="$PATH:/opt/commander"
 
 # Install Silicon Labs Configurator (slc)
+ARG SLC_CLI_SHA256SUM=da4faa09ef4cbe385da71e5b95a4e444666cf4aaca6066b1095ca13bf5ebf233
 RUN \
     curl -O https://www.silabs.com/documents/login/software/slc_cli_linux.zip \
+    && echo "${SLC_CLI_SHA256SUM}  slc_cli_linux.zip" | sha256sum -c \
     && unzip -q -d /opt slc_cli_linux.zip \
     && rm slc_cli_linux.zip
 
 ENV PATH="$PATH:/opt/slc_cli"
 
 # GCC Embedded Toolchain 12.2.rel1 (for Gecko SDK 4.4.0+)
+ARG GCC_TOOLCHAIN_SHA256SUM=84be93d0f9e96a15addd490b6e237f588c641c8afdf90e7610a628007fc96867
 RUN \
     curl -O https://armkeil.blob.core.windows.net/developer/Files/downloads/gnu/12.2.rel1/binrel/arm-gnu-toolchain-12.2.rel1-x86_64-arm-none-eabi.tar.xz \
+    && echo "${GCC_TOOLCHAIN_SHA256SUM}  arm-gnu-toolchain-12.2.rel1-x86_64-arm-none-eabi.tar.xz" | sha256sum -c \
     && tar -C /opt -xf arm-gnu-toolchain-12.2.rel1-x86_64-arm-none-eabi.tar.xz \
     && rm arm-gnu-toolchain-12.2.rel1-x86_64-arm-none-eabi.tar.xz
 
 # Simplicity SDK 2024.6.2
+ARG SIMPLICITY_SDK_SHA256SUM=7e4337c7cc68262dd3a83c8528095774634a0478d40b1c1fd2b462e86236af8a
 RUN \
     curl -o simplicity_sdk_2024.6.2.zip -L https://github.com/SiliconLabs/simplicity_sdk/releases/download/v2024.6.2/gecko-sdk.zip \
+    && echo "${SIMPLICITY_SDK_SHA256SUM}  simplicity_sdk_2024.6.2.zip" | sha256sum -c \
     && unzip -q -d simplicity_sdk_2024.6.2 simplicity_sdk_2024.6.2.zip \
     && rm simplicity_sdk_2024.6.2.zip
 
 # Gecko SDK 4.4.4
+ARG GECKO_SDK_SHA256SUM=831ec7c564df4392b18a8cc8ceb228c114dc3bec604be75807961a4289ee9b20
 RUN \
     curl -o gecko_sdk_4.4.4.zip -L https://github.com/SiliconLabs/gecko_sdk/releases/download/v4.4.4/gecko-sdk.zip \
+    && echo "${GECKO_SDK_SHA256SUM}  gecko_sdk_4.4.4.zip" | sha256sum -c \
     && unzip -q -d gecko_sdk_4.4.4 gecko_sdk_4.4.4.zip \
     && rm gecko_sdk_4.4.4.zip
 
 # ZCL Advanced Platform (ZAP) v2024.09.27
+ARG ZAP_SHA256SUM=22beeae3cf33b04792be379261d68695b5c96986d3b80700c22b1348f4c0421e
 RUN \
     curl -o zap_2024.09.27.zip -L https://github.com/project-chip/zap/releases/download/v2024.09.27/zap-linux-x64.zip \
+    && echo "${ZAP_SHA256SUM}  zap_2024.09.27.zip" | sha256sum -c \
     && unzip -q -d /opt/zap zap_2024.09.27.zip \
     && rm zap_2024.09.27.zip
 
@@ -81,5 +94,12 @@ ARG USER_GID=$USER_UID
 RUN groupadd --gid $USER_GID $USERNAME \
     && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME
 
+RUN mkdir -p /build_dir /outputs
+RUN chown $USERNAME:$USERNAME \
+    /gecko_sdk_* \
+    /simplicity_sdk_* \
+    /build_dir \
+    /outputs
+
 USER $USERNAME
 WORKDIR /build

Makefile

@@ -0,0 +1,75 @@
+TOPDIR = $(shell pwd)
+DOCKER_CHECK := $(shell command -v docker 2> /dev/null)
+PODMAN_CHECK := $(shell command -v podman 2> /dev/null)
+
+ifdef PODMAN_CHECK
+    CONTAINER_ENGINE ?= podman
+else ifdef DOCKER_CHECK
+    CONTAINER_ENGINE ?= docker
+else
+    $(error "No container engine found, please install docker or podman")
+endif
+
+CONTAINER_NAME ?= silabs-firmware-builder
+CONTAINER_USER_GROUP ?= $(shell id -u):$(shell id -g)
+
+ifeq ($(CONTAINER_ENGINE),docker)
+    VOLUME_OPTS=
+else
+    VOLUME_OPTS=:z
+endif
+
+define run_in_container
+	$(CONTAINER_ENGINE) \
+		run --rm \
+		--user $(CONTAINER_USER_GROUP) \
+		-v $(TOPDIR):/build$(VOLUME_OPTS) \
+		-v $(TOPDIR)/outputs:/outputs$(VOLUME_OPTS) \
+		-v $(TOPDIR)/build_dir:/build_dir$(VOLUME_OPTS) \
+		$(CONTAINER_NAME)
+endef
+
+MANIFESTS ?= $(shell find manifests -type f \( -name "*.yaml" -o -name "*.yml" \) -print)
+
+all: build_container build_firmware
+
+help:
+	@echo "Usage: make [all|build_container|build_firmware]"
+	@echo ""
+	@echo "Targets:"
+	@echo "  all             Build container and firmware"
+	@echo "  build_container Build container"
+	@echo "  build_firmware  Build firmware"
+	@echo "  help            Show this help message"
+	@echo ""
+	@echo "Options:"
+	@echo "  build_firmware MANIFESTS=<path>  Override default manifest files (default: all .yaml/.yml files in manifests/)"
+	@echo ""
+	@echo "Examples:"
+	@echo "  # Build the container image"
+	@echo "  make build_container"
+	@echo ""
+	@echo "  # Build all firmware manifests"
+	@echo "  make build_firmware"
+	@echo ""
+	@echo "  # Build a specific firmware manifest"
+	@echo "  make build_firmware MANIFESTS=manifests/nabucasa/yellow_bootloader.yaml"
+	@echo ""
+
+./outputs ./build_dir:
+	mkdir -p $@
+ifneq ($(CONTAINER_ENGINE),docker)
+	$(CONTAINER_ENGINE) unshare chown -R $(shell id -u):$(shell id -g) $@
+endif
+
+build_container:
+	$(CONTAINER_ENGINE) build -t $(CONTAINER_NAME) .
+
+build_firmware: ./outputs ./build_dir
+	$(run_in_container) \
+		bash -c " \
+			build_firmware.sh \
+			--build-dir /build_dir \
+			--output-dir /outputs \
+			$(foreach manifest,$(MANIFESTS),--manifest $(manifest)) \
+		"

README.md

@@ -90,3 +90,117 @@ python tools/build_project.py \
 ```
 
 Once the build is complete, the firmwares will be in the `output` directory.
+
+## Building with a container (for development)
+
+This is a convenience GNU Make  based wrapper around the build process that is being used on the GitHub Actions CI pipeline, but can also be used for local development.
+
+### Prerequisites
+
+- [GNU Make](https://www.gnu.org/software/make/)
+- [Docker](https://docs.docker.com/get-docker/) or [Podman](https://podman-desktop.io/)
+
+| Prerequisite | macOS | Windows | Debian/Ubuntu | Fedora |
+|-------------|--------|---------|---------------|---------|
+| GNU Make | `brew install make` | Via [Chocolatey](https://chocolatey.org/): `choco install make` | `sudo apt install make` | `sudo dnf install make` |
+| Docker | Download [Docker Desktop](https://www.docker.com/products/docker-desktop/) | Download [Docker Desktop](https://www.docker.com/products/docker-desktop/) | `sudo apt install docker.io` | `sudo dnf install docker-ce docker-ce-cli containerd.io` |
+| Podman | `brew install podman` | Download [Podman Desktop](https://podman-desktop.io/downloads) | `sudo apt install podman` | `sudo dnf install podman` |
+
+### Usage
+
+#### Help
+
+Provides a list of commands and options.
+
+```bash
+make help
+
+Usage: make [all|build_container|build_firmware]
+
+Targets:
+  all             Build container and firmware
+  build_container Build container
+  build_firmware  Build firmware
+  help            Show this help message
+
+Options:
+  build_firmware MANIFESTS=<path>  Override default manifest files (default: all .yaml/.yml files in manifests/)
+
+Examples:
+  # Build the container image
+  make build_container
+
+  # Build all firmware manifests
+  make build_firmware
+
+  # Build a specific firmware manifest
+  make build_firmware MANIFESTS=manifests/nabucasa/yellow_bootloader.yaml
+```
+
+#### Build everything
+
+Builds the container image and all available firmware manifests.
+
+```bash
+make
+```
+
+Once this command completes, the firmwares will be in the `outputs` directory.
+
+```bash
+ls -w 80 outputs | head -3
+skyconnect_bootloader_2.4.2.gbl
+skyconnect_bootloader_2.4.2.hex
+skyconnect_bootloader_2.4.2.out
+```
+
+#### Build the container
+
+Builds only the container image.
+
+```bash
+make build_container
+```
+
+#### Build all available firmware manifests
+
+Builds all available firmware manifests in the `manifests` directory.
+
+```bash
+make build_firmware
+```
+
+#### Build a specific firmware manifest
+
+Builds a specific firmware manifest by providing the path to the manifest file.
+
+```bash
+make build_firmware MANIFESTS=manifests/nabucasa/yellow_openthread_ncp.yaml
+```
+
+Once this command completes, the firmwares will be in the `outputs` directory.
+
+```bash
+ls -w 80 outputs
+yellow_openthread_rcp_2.4.4.0_GitHub-7074a43e4_gsdk_4.4.4.gbl
+yellow_openthread_rcp_2.4.4.0_GitHub-7074a43e4_gsdk_4.4.4.hex
+yellow_openthread_rcp_2.4.4.0_GitHub-7074a43e4_gsdk_4.4.4.out
+```
+
+#### Build with a custom container image
+
+Builds the firmware with a custom container image by providing the container image name.
+
+```bash
+make build_firmware CONTAINER_NAME=ghcr.io/nabucasa/silabs-firmware-builder
+```
+
+### Makefile variables
+
+The following variables can be customized when running make commands:
+
+| Variable | Default Value | Description |
+|----------|---------------|-------------|
+| CONTAINER_NAME | silabs-firmware-builder | Name of the container image to build/use |
+| CONTAINER_ENGINE | docker | Container engine to use (docker or podman) |
+| MANIFESTS | every file in `manifests`  directory| Which firmware manifests to build |