package ssbl files together with px4 build container

.github/workflows/tiiuae-pixhawk-and-saluki.yaml

@@ -153,7 +153,9 @@ jobs:
     name: upload px4fwupdater to docker registry
     runs-on: ubuntu-latest
     env:
-      FPGA_DIR: fpga
+      BUILD_DIR: combine
+      FPGA_DIR: combine/fpga
+      BOOTLOADER_DIR: combine/bootloader
     needs:
       - px4fwupdater
       - variables
@@ -162,6 +164,8 @@ jobs:
         uses: actions/checkout@v4
         with:
           path: px4-firmware
+          submodules: recursive
+          token: ${{ secrets.GH_REPO_TOKEN }}
           fetch-depth: 0
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -193,14 +197,46 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: fetch fpga files
         run: |
+          set -eux
+
+          pip install check-jsonschema
+
           echo "fetching fpga files from ${{ env.saluki_fpga_repo }}"
           px4-firmware/Tools/fetch-fpga-files.sh ${{ env.saluki_fpga_repo }}:${{ needs.variables.outputs.saluki_v2_fpga_version }} saluki_v2 ${{ env.FPGA_DIR }}/saluki-v2
           px4-firmware/Tools/fetch-fpga-files.sh ${{ env.saluki_fpga_repo }}:${{ needs.variables.outputs.saluki_v3_fpga_version }} saluki_v3 ${{ env.FPGA_DIR }}/saluki-v3
           px4-firmware/Tools/fetch-fpga-files.sh ${{ env.saluki_fpga_repo }}:${{ needs.variables.outputs.saluki_pi_fpga_version }} saluki_pi ${{ env.FPGA_DIR }}/saluki-pi
           px4-firmware/Tools/fetch-fpga-files.sh ${{ env.saluki_fpga_repo }}:${{ needs.variables.outputs.saluki_fmu2_fpga_version }} saluki_fmu2 ${{ env.FPGA_DIR }}/saluki-fmu2
 
-          # combine fpga json files to one file
-          python px4-firmware/Tools/collect_json_files.py ${{ env.FPGA_DIR }} ${{ env.FPGA_DIR }}/fpga-file-info.json
+          echo "find out bootloader version from fpga files"
+          saluki_v2_bootloader_container=$(jq -r '.["fpga-files"][] | select(.type | contains("fpga_job")).bootloader_container' ${{ env.FPGA_DIR }}/saluki-v2/saluki_file_info.json)
+          saluki_v3_bootloader_container=$(jq -r '.["fpga-files"][] | select(.type | contains("fpga_job")).bootloader_container' ${{ env.FPGA_DIR }}/saluki-v3/saluki_file_info.json)
+          saluki_pi_bootloader_container=$(jq -r '.["fpga-files"][] | select(.type | contains("fpga_job")).bootloader_container' ${{ env.FPGA_DIR }}/saluki-pi/saluki_file_info.json)
+          saluki_fmu2_bootloader_container=$(jq -r '.["fpga-files"][] | select(.type | contains("fpga_job")).bootloader_container' ${{ env.FPGA_DIR }}/saluki-fmu2/saluki_file_info.json)
+
+          echo "fetch bootloader files"
+          if [[ -n "${saluki_v2_bootloader_container}" ]]; then
+            px4-firmware/Tools/fetch-ssbl-files.sh ${saluki_v2_bootloader_container} saluki-v2 ${{ env.BOOTLOADER_DIR }}/saluki-v2
+            # fetch custom signer bootloader
+            px4-firmware/Tools/fetch-ssbl-files.sh ${saluki_v2_bootloader_container} saluki-v2_custom ${{ env.BOOTLOADER_DIR }}/saluki-v2_custom
+          fi
+          if [[ -n "${saluki_v3_bootloader_container}" ]]; then
+            px4-firmware/Tools/fetch-ssbl-files.sh ${saluki_v3_bootloader_container} saluki-v3 ${{ env.BOOTLOADER_DIR }}/saluki-v3
+            # fetch custom signer bootloader
+            px4-firmware/Tools/fetch-ssbl-files.sh ${saluki_v3_bootloader_container} saluki-v3_custom ${{ env.BOOTLOADER_DIR }}/saluki-v3_custom
+          fi
+          if [[ -n "${saluki_pi_bootloader_container}" ]]; then
+            px4-firmware/Tools/fetch-ssbl-files.sh ${saluki_pi_bootloader_container} saluki-pi ${{ env.BOOTLOADER_DIR }}/saluki-pi
+            # fetch custom signer bootloader
+            px4-firmware/Tools/fetch-ssbl-files.sh ${saluki_pi_bootloader_container} saluki-pi_custom ${{ env.BOOTLOADER_DIR }}/saluki-pi_custom
+          fi
+          if [[ -n "${saluki_fmu2_bootloader_container}" ]]; then
+            px4-firmware/Tools/fetch-ssbl-files.sh ${saluki_fmu2_bootloader_container} saluki-fmu2 ${{ env.BOOTLOADER_DIR }}/saluki-fmu2
+            # fetch custom signer bootloader
+            px4-firmware/Tools/fetch-ssbl-files.sh ${saluki_fmu2_bootloader_container} saluki-fmu2_custom ${{ env.BOOTLOADER_DIR }}/saluki-fmu2_custom
+          fi
+
+          # combine fpga+BL json files to one file
+          python px4-firmware/Tools/collect_json_files.py ${{ env.BUILD_DIR }} ${{ env.BUILD_DIR }}/fpga-file-info.json
 
           # generate saluki file info json with project basic info
           # fetched info from git so has to be ran inside the repo
@@ -209,7 +245,10 @@ jobs:
           popd
 
           # combine saluki file info json with fpga json and px4 json files to one file
-          python px4-firmware/Tools/compile_px4_build_info.py saluki_file_info.json bin ${{ env.FPGA_DIR }}/fpga-file-info.json build_basic_info.json
+          python px4-firmware/Tools/compile_px4_build_info.py saluki_file_info.json bin ${{ env.BUILD_DIR }}/fpga-file-info.json build_basic_info.json
+
+          # check saluki_file_info.json is valid json
+          check-jsonschema --schemafile px4-firmware/Tools/saluki_packaging/saluki_file_info-schema.json saluki_file_info.json
 
       - name: Firmware flasher - Build and push
         uses: docker/build-push-action@v5
@@ -222,7 +261,7 @@ jobs:
           tags: ${{ steps.containermeta.outputs.tags }}
           labels: ${{ steps.containermeta.outputs.labels }}
           build-args: |
-            "saluki_fpga_directory=${{ env.FPGA_DIR }}"
+            "saluki_fpga_directory=${{ env.BUILD_DIR }}"
       - name: Build overview
         run: |
           echo "### Build overview:" >> $GITHUB_STEP_SUMMARY

.gitmodules

@@ -112,3 +112,6 @@
 [submodule "src/drivers/imx9_keystore"]
 	path = src/drivers/imx9_keystore
 	url = [email protected]:tiiuae/imx9_keystore.git
+[submodule "Tools/packaging"]
+	path = Tools/saluki_packaging
+	url = [email protected]:tiiuae/saluki_low_level_sw_build.git

Tools/collect_json_files.py

@@ -30,23 +30,30 @@
     with open(file, 'r') as f:
         data = json.load(f)
 
-        # files array has to be merged separately
+        # fpga-files array has to be merged separately
         fpga_files_tmp_data = json_data.get("fpga-files", []) + data.get("fpga-files", [])
 
-        # WORKAROUND: different naming schemes used in different repos
-        # fpga and bootloader uses for example saluki_pi (underscore)
-        # px4 uses saluki-pi (dash)
         # replace all saluki_ with saluki- in fpga-files[].hw
         for fpga_file in fpga_files_tmp_data:
             if fpga_file.get("hw").startswith("saluki_"):
                 fpga_file["hw"] = fpga_file["hw"].replace("saluki_", "saluki-")
 
+        # bootloader-files array has to be merged separately
+        bl_files_tmp_data = json_data.get("bl-files", []) + data.get("bl-files", [])
+
+        # replace all saluki_ with saluki- in bl-files[].hw
+        for bl_file in bl_files_tmp_data:
+            if bl_file.get("hw").startswith("saluki_"):
+                bl_file["hw"] = bl_file["hw"].replace("saluki_", "saluki-")
+
         # combine all json data into one
         json_data = {**json_data, **data}
 
         # add fpga-files from temporary merged array
         json_data["fpga-files"] = fpga_files_tmp_data
 
+        # add fpga-files from temporary merged array
+        json_data["bl-files"] = bl_files_tmp_data
 
 print( "{}: this is the combined json data".format(sys.argv[0]))
 print (json.dumps(json_data, indent=4, sort_keys=True))

Tools/compile_px4_build_info.py

@@ -45,7 +45,17 @@
 # read fpga json file
 with open(fpga_json_file, 'r') as f:
   fpga_json_data = json.load(f)
+  # add fpga-files array to files array
   fpga_json_data['files'] = fpga_json_data.pop('fpga-files')
+  # add bl-files array to files array
+  fpga_json_data['files'].extend(fpga_json_data.pop('bl-files'))
+  # add sha field to fpga_sha
+  fpga_json_data['fpga_sha'] = fpga_json_data.pop('sha', '')
+  # add build_url field to fpga_build_url
+  fpga_json_data['fpga_build_url'] = fpga_json_data.pop('build_url', '')
+  # add reponame field to fpga_reponame
+  fpga_json_data['fpga_reponame'] = fpga_json_data.pop('reponame', '')
+
   print("{}: fpga json data".format(sys.argv[0], file))
   print(json.dumps(fpga_json_data, indent=4, sort_keys=True))
 

Tools/fetch-fpga-files.sh

@@ -1,4 +1,4 @@
-set -eu
+#!/bin/bash -eu
 
 # input parameters:
 # 1: repo url
@@ -35,8 +35,10 @@ else
 	echo "using info from ${fileinfo_json_out}"
 	# find out info from json
 	# files to be used from fileinfo_json files[] -> filename
-	# product=$(echo ${product}|sed 's/_/-/g')
-	filename=$(jq '.files[] | select(.hw=="'${product}'").filename' ${fileinfo_json_out}|sed 's/"//g')
+	#product=$(echo ${product}|sed 's/_/-/g')
+	filename=$(jq -r '.files[] | select(.hw=="'${product}'").filename' ${fileinfo_json_out})
+
+  echo "filet: ${filename}"
 
   tmp_reponame=$(jq '.reponame' ${fileinfo_json_out})
   tmp_sha=$(jq '.sha' ${fileinfo_json_out})

Tools/fetch-ssbl-files.sh

@@ -0,0 +1,74 @@
+#!/bin/bash -eu
+
+# this script fetches second stage bootloader files from a docker container
+# ssbl files are not necessary available so it might not extract anything
+
+# input parameters:
+# 1: bootloader repo url
+# 2: PRODUCT
+# 3: output dir
+
+if [ $# -ne 3 ]; then
+  echo "Error: ${0} requires 3 parameters"
+  echo "Usage: ${0} <repo url> <PRODUCT> <output dir>"
+  exit 1
+fi
+
+BL_REPO_URL="${1:-}"
+PRODUCT="${2:-}"
+OUTPUT_DIR="${3:-}"
+TMP_CONTAINER_NAME=tmp_bootloader_container_$(date +%s)
+SALUKI_FILE_INFO="saluki_file_info.json"
+FILEINFO_JSON_OUT=${OUTPUT_DIR}/${SALUKI_FILE_INFO}
+
+function cleanup_and_exit {
+    docker stop ${TMP_CONTAINER_NAME}
+    docker rm ${TMP_CONTAINER_NAME}
+    exit 0
+}
+
+# run cleanup always on exit
+trap cleanup_and_exit EXIT
+
+echo "fetching bootloader files from ${BL_REPO_URL}"
+
+docker run -d --pull always --name ${TMP_CONTAINER_NAME} ${BL_REPO_URL}
+
+# try to get the build info json
+mkdir -p ${OUTPUT_DIR}
+docker cp ${TMP_CONTAINER_NAME}:/${SALUKI_FILE_INFO} ${FILEINFO_JSON_OUT} || true
+
+# if saluki build info doesnt exist in bootloader, it has to be not split version
+if [ ! -f ${FILEINFO_JSON_OUT} ]; then
+	echo "bootloader not split version"
+    exit 0
+else
+	echo "using info from ${FILEINFO_JSON_OUT}"
+	# find out if the 2 stage bootloader files are available
+	# files should have stage defined: fileinfo_json files[] -> stage
+	PRODUCT=$(echo ${PRODUCT}|sed 's/_/-/g')
+	filename=$(jq '.files[] | select(.hw=="'${PRODUCT}'" and .stage=="ssbl").filename' ${FILEINFO_JSON_OUT}|sed 's/"//g')
+
+    tmp_bl_reponame=$(jq '.reponame' ${FILEINFO_JSON_OUT})
+    tmp_bl_sha=$(jq '.sha' ${FILEINFO_JSON_OUT})
+    tmp_bl_build_url=$(jq '.build_url' ${FILEINFO_JSON_OUT})
+
+    # filter files for this product and append $output_dir to .files[].filename
+    # file path is appended because the files are copied to a different directory so it will match the fileinfo.json
+    tmp_files=$(jq '.files[] | select(.hw=="'${PRODUCT}'" and .stage=="ssbl") | .filename = "/firmware/'${OUTPUT_DIR}'"+.filename' ${FILEINFO_JSON_OUT}|jq -s .)
+
+    if [ -z "${tmp_files}" ]; then
+        echo "no ssbl files found"
+        exit 0
+    fi
+
+    tmp_json="{\"${PRODUCT}_bootloader_reponame\":${tmp_bl_reponame},\"${PRODUCT}_bl_sha\":${tmp_bl_sha},\"${PRODUCT}_bl_build_url\":${tmp_bl_build_url},\"bl-files\":${tmp_files}}"
+    echo $(echo $tmp_json|jq .)
+    echo $tmp_json|jq . > ${FILEINFO_JSON_OUT}
+
+    # copy each file
+    echo "make target dir (dirname of) ${OUTPUT_DIR}/${filename}"
+    mkdir -p $(dirname ${OUTPUT_DIR}/${filename})
+    echo "copying ${filename} to ${OUTPUT_DIR}/${filename}"
+    docker cp ${TMP_CONTAINER_NAME}:${filename} ${OUTPUT_DIR}/${filename}
+fi

Tools/generate_basic_build_info.sh

@@ -34,6 +34,9 @@ build_info["reponame"]=${repo}
 build_info["sha"]=${sha}
 build_info["px4_firmware_sha"]=${px4_sha}
 build_info["build_url"]=${build_url}
+build_info["px4_build_url"]=${build_url}
+build_info["px4_reponame"]=${repo}
+build_info["px4_sha"]=${sha}
 
 # loop thru associative array and print key value pairs
 json_info="{"

Tools/px_uploader.Dockerfile

@@ -5,7 +5,7 @@ ARG SALUKI_FILE_INFO_JSON=saluki_file_info.json
 ARG PX4_EXPORT_DIR
 ENV PX4_EXPORT_DIR=$PX4_EXPORT_DIR
 
-COPY $saluki_fpga_directory /firmware/fpga
+COPY $saluki_fpga_directory /firmware/combine
 COPY $SALUKI_FILE_INFO_JSON /$SALUKI_FILE_INFO_JSON
 WORKDIR /firmware
 

clone_public.sh

@@ -15,6 +15,7 @@ do
   [[ "${repo}" == *rust_px4_nuttx ]] || \
   [[ "${repo}" == *rust_module_example ]] || \
   [[ "${repo}" == *secure_udp_proxy ]] || \
+  [[ "${repo}" == *saluki_packaging ]] || \
   [[ "${repo}" == *process ]] && continue
   git submodule update --init --recursive "${repo}"
 done <<< "$(git submodule status | awk '{print $2}')"