From d0ab98ef70a252d6d10d649a7f7010cc8deb82eb Mon Sep 17 00:00:00 2001 From: Timo Sairiala Date: Fri, 29 Nov 2024 14:14:36 +0200 Subject: [PATCH] package ssbl files together with px4 build container --- .../workflows/tiiuae-pixhawk-and-saluki.yaml | 49 ++++++++++-- .gitmodules | 3 + Tools/collect_json_files.py | 15 +++- Tools/compile_px4_build_info.py | 10 +++ Tools/fetch-fpga-files.sh | 8 +- Tools/fetch-ssbl-files.sh | 74 +++++++++++++++++++ Tools/generate_basic_build_info.sh | 3 + Tools/px_uploader.Dockerfile | 2 +- Tools/saluki_packaging | 1 + clone_public.sh | 1 + 10 files changed, 153 insertions(+), 13 deletions(-) create mode 100755 Tools/fetch-ssbl-files.sh create mode 160000 Tools/saluki_packaging diff --git a/.github/workflows/tiiuae-pixhawk-and-saluki.yaml b/.github/workflows/tiiuae-pixhawk-and-saluki.yaml index 60ca97594072..ea577c956acc 100644 --- a/.github/workflows/tiiuae-pixhawk-and-saluki.yaml +++ b/.github/workflows/tiiuae-pixhawk-and-saluki.yaml @@ -153,7 +153,9 @@ jobs: name: upload px4fwupdater to docker registry runs-on: ubuntu-latest env: - FPGA_DIR: fpga + BUILD_DIR: combine + FPGA_DIR: combine/fpga + BOOTLOADER_DIR: combine/bootloader needs: - px4fwupdater - variables @@ -162,6 +164,8 @@ jobs: uses: actions/checkout@v4 with: path: px4-firmware + submodules: recursive + token: ${{ secrets.GH_REPO_TOKEN }} fetch-depth: 0 - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -193,14 +197,46 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: fetch fpga files run: | + set -eux + + pip install check-jsonschema + echo "fetching fpga files from ${{ env.saluki_fpga_repo }}" px4-firmware/Tools/fetch-fpga-files.sh ${{ env.saluki_fpga_repo }}:${{ needs.variables.outputs.saluki_v2_fpga_version }} saluki_v2 ${{ env.FPGA_DIR }}/saluki-v2 px4-firmware/Tools/fetch-fpga-files.sh ${{ env.saluki_fpga_repo }}:${{ needs.variables.outputs.saluki_v3_fpga_version }} saluki_v3 ${{ env.FPGA_DIR }}/saluki-v3 px4-firmware/Tools/fetch-fpga-files.sh ${{ env.saluki_fpga_repo }}:${{ needs.variables.outputs.saluki_pi_fpga_version }} saluki_pi ${{ env.FPGA_DIR }}/saluki-pi px4-firmware/Tools/fetch-fpga-files.sh ${{ env.saluki_fpga_repo }}:${{ needs.variables.outputs.saluki_fmu2_fpga_version }} saluki_fmu2 ${{ env.FPGA_DIR }}/saluki-fmu2 - # combine fpga json files to one file - python px4-firmware/Tools/collect_json_files.py ${{ env.FPGA_DIR }} ${{ env.FPGA_DIR }}/fpga-file-info.json + echo "find out bootloader version from fpga files" + saluki_v2_bootloader_container=$(jq -r '.["fpga-files"][] | select(.type | contains("fpga_job")).bootloader_container' ${{ env.FPGA_DIR }}/saluki-v2/saluki_file_info.json) + saluki_v3_bootloader_container=$(jq -r '.["fpga-files"][] | select(.type | contains("fpga_job")).bootloader_container' ${{ env.FPGA_DIR }}/saluki-v3/saluki_file_info.json) + saluki_pi_bootloader_container=$(jq -r '.["fpga-files"][] | select(.type | contains("fpga_job")).bootloader_container' ${{ env.FPGA_DIR }}/saluki-pi/saluki_file_info.json) + saluki_fmu2_bootloader_container=$(jq -r '.["fpga-files"][] | select(.type | contains("fpga_job")).bootloader_container' ${{ env.FPGA_DIR }}/saluki-fmu2/saluki_file_info.json) + + echo "fetch bootloader files" + if [[ -n "${saluki_v2_bootloader_container}" ]]; then + px4-firmware/Tools/fetch-ssbl-files.sh ${saluki_v2_bootloader_container} saluki-v2 ${{ env.BOOTLOADER_DIR }}/saluki-v2 + # fetch custom signer bootloader + px4-firmware/Tools/fetch-ssbl-files.sh ${saluki_v2_bootloader_container} saluki-v2_custom ${{ env.BOOTLOADER_DIR }}/saluki-v2_custom + fi + if [[ -n "${saluki_v3_bootloader_container}" ]]; then + px4-firmware/Tools/fetch-ssbl-files.sh ${saluki_v3_bootloader_container} saluki-v3 ${{ env.BOOTLOADER_DIR }}/saluki-v3 + # fetch custom signer bootloader + px4-firmware/Tools/fetch-ssbl-files.sh ${saluki_v3_bootloader_container} saluki-v3_custom ${{ env.BOOTLOADER_DIR }}/saluki-v3_custom + fi + if [[ -n "${saluki_pi_bootloader_container}" ]]; then + px4-firmware/Tools/fetch-ssbl-files.sh ${saluki_pi_bootloader_container} saluki-pi ${{ env.BOOTLOADER_DIR }}/saluki-pi + # fetch custom signer bootloader + px4-firmware/Tools/fetch-ssbl-files.sh ${saluki_pi_bootloader_container} saluki-pi_custom ${{ env.BOOTLOADER_DIR }}/saluki-pi_custom + fi + if [[ -n "${saluki_fmu2_bootloader_container}" ]]; then + px4-firmware/Tools/fetch-ssbl-files.sh ${saluki_fmu2_bootloader_container} saluki-fmu2 ${{ env.BOOTLOADER_DIR }}/saluki-fmu2 + # fetch custom signer bootloader + px4-firmware/Tools/fetch-ssbl-files.sh ${saluki_fmu2_bootloader_container} saluki-fmu2_custom ${{ env.BOOTLOADER_DIR }}/saluki-fmu2_custom + fi + + # combine fpga+BL json files to one file + python px4-firmware/Tools/collect_json_files.py ${{ env.BUILD_DIR }} ${{ env.BUILD_DIR }}/fpga-file-info.json # generate saluki file info json with project basic info # fetched info from git so has to be ran inside the repo @@ -209,7 +245,10 @@ jobs: popd # combine saluki file info json with fpga json and px4 json files to one file - python px4-firmware/Tools/compile_px4_build_info.py saluki_file_info.json bin ${{ env.FPGA_DIR }}/fpga-file-info.json build_basic_info.json + python px4-firmware/Tools/compile_px4_build_info.py saluki_file_info.json bin ${{ env.BUILD_DIR }}/fpga-file-info.json build_basic_info.json + + # check saluki_file_info.json is valid json + check-jsonschema --schemafile px4-firmware/Tools/saluki_packaging/saluki_file_info-schema.json saluki_file_info.json - name: Firmware flasher - Build and push uses: docker/build-push-action@v5 @@ -222,7 +261,7 @@ jobs: tags: ${{ steps.containermeta.outputs.tags }} labels: ${{ steps.containermeta.outputs.labels }} build-args: | - "saluki_fpga_directory=${{ env.FPGA_DIR }}" + "saluki_fpga_directory=${{ env.BUILD_DIR }}" - name: Build overview run: | echo "### Build overview:" >> $GITHUB_STEP_SUMMARY diff --git a/.gitmodules b/.gitmodules index 25bf39c695f3..1c7dd3256332 100644 --- a/.gitmodules +++ b/.gitmodules @@ -112,3 +112,6 @@ [submodule "src/drivers/imx9_keystore"] path = src/drivers/imx9_keystore url = git@github.com:tiiuae/imx9_keystore.git +[submodule "Tools/packaging"] + path = Tools/saluki_packaging + url = git@github.com:tiiuae/saluki_low_level_sw_build.git diff --git a/Tools/collect_json_files.py b/Tools/collect_json_files.py index 5a994b3c6663..ade55d6f6ed2 100644 --- a/Tools/collect_json_files.py +++ b/Tools/collect_json_files.py @@ -30,23 +30,30 @@ with open(file, 'r') as f: data = json.load(f) - # files array has to be merged separately + # fpga-files array has to be merged separately fpga_files_tmp_data = json_data.get("fpga-files", []) + data.get("fpga-files", []) - # WORKAROUND: different naming schemes used in different repos - # fpga and bootloader uses for example saluki_pi (underscore) - # px4 uses saluki-pi (dash) # replace all saluki_ with saluki- in fpga-files[].hw for fpga_file in fpga_files_tmp_data: if fpga_file.get("hw").startswith("saluki_"): fpga_file["hw"] = fpga_file["hw"].replace("saluki_", "saluki-") + # bootloader-files array has to be merged separately + bl_files_tmp_data = json_data.get("bl-files", []) + data.get("bl-files", []) + + # replace all saluki_ with saluki- in bl-files[].hw + for bl_file in bl_files_tmp_data: + if bl_file.get("hw").startswith("saluki_"): + bl_file["hw"] = bl_file["hw"].replace("saluki_", "saluki-") + # combine all json data into one json_data = {**json_data, **data} # add fpga-files from temporary merged array json_data["fpga-files"] = fpga_files_tmp_data + # add fpga-files from temporary merged array + json_data["bl-files"] = bl_files_tmp_data print( "{}: this is the combined json data".format(sys.argv[0])) print (json.dumps(json_data, indent=4, sort_keys=True)) diff --git a/Tools/compile_px4_build_info.py b/Tools/compile_px4_build_info.py index d3acf1f67570..a3a74c30e5b5 100644 --- a/Tools/compile_px4_build_info.py +++ b/Tools/compile_px4_build_info.py @@ -45,7 +45,17 @@ # read fpga json file with open(fpga_json_file, 'r') as f: fpga_json_data = json.load(f) + # add fpga-files array to files array fpga_json_data['files'] = fpga_json_data.pop('fpga-files') + # add bl-files array to files array + fpga_json_data['files'].extend(fpga_json_data.pop('bl-files')) + # add sha field to fpga_sha + fpga_json_data['fpga_sha'] = fpga_json_data.pop('sha', '') + # add build_url field to fpga_build_url + fpga_json_data['fpga_build_url'] = fpga_json_data.pop('build_url', '') + # add reponame field to fpga_reponame + fpga_json_data['fpga_reponame'] = fpga_json_data.pop('reponame', '') + print("{}: fpga json data".format(sys.argv[0], file)) print(json.dumps(fpga_json_data, indent=4, sort_keys=True)) diff --git a/Tools/fetch-fpga-files.sh b/Tools/fetch-fpga-files.sh index 0a96faa8b61a..86699c658d5f 100755 --- a/Tools/fetch-fpga-files.sh +++ b/Tools/fetch-fpga-files.sh @@ -1,4 +1,4 @@ -set -eu +#!/bin/bash -eu # input parameters: # 1: repo url @@ -35,8 +35,10 @@ else echo "using info from ${fileinfo_json_out}" # find out info from json # files to be used from fileinfo_json files[] -> filename - # product=$(echo ${product}|sed 's/_/-/g') - filename=$(jq '.files[] | select(.hw=="'${product}'").filename' ${fileinfo_json_out}|sed 's/"//g') + #product=$(echo ${product}|sed 's/_/-/g') + filename=$(jq -r '.files[] | select(.hw=="'${product}'").filename' ${fileinfo_json_out}) + + echo "filet: ${filename}" tmp_reponame=$(jq '.reponame' ${fileinfo_json_out}) tmp_sha=$(jq '.sha' ${fileinfo_json_out}) diff --git a/Tools/fetch-ssbl-files.sh b/Tools/fetch-ssbl-files.sh new file mode 100755 index 000000000000..4f0fc53c4626 --- /dev/null +++ b/Tools/fetch-ssbl-files.sh @@ -0,0 +1,74 @@ +#!/bin/bash -eu + +# this script fetches second stage bootloader files from a docker container +# ssbl files are not necessary available so it might not extract anything + +# input parameters: +# 1: bootloader repo url +# 2: PRODUCT +# 3: output dir + +if [ $# -ne 3 ]; then + echo "Error: ${0} requires 3 parameters" + echo "Usage: ${0} " + exit 1 +fi + +BL_REPO_URL="${1:-}" +PRODUCT="${2:-}" +OUTPUT_DIR="${3:-}" +TMP_CONTAINER_NAME=tmp_bootloader_container_$(date +%s) +SALUKI_FILE_INFO="saluki_file_info.json" +FILEINFO_JSON_OUT=${OUTPUT_DIR}/${SALUKI_FILE_INFO} + +function cleanup_and_exit { + docker stop ${TMP_CONTAINER_NAME} + docker rm ${TMP_CONTAINER_NAME} + exit 0 +} + +# run cleanup always on exit +trap cleanup_and_exit EXIT + +echo "fetching bootloader files from ${BL_REPO_URL}" + +docker run -d --pull always --name ${TMP_CONTAINER_NAME} ${BL_REPO_URL} + +# try to get the build info json +mkdir -p ${OUTPUT_DIR} +docker cp ${TMP_CONTAINER_NAME}:/${SALUKI_FILE_INFO} ${FILEINFO_JSON_OUT} || true + +# if saluki build info doesnt exist in bootloader, it has to be not split version +if [ ! -f ${FILEINFO_JSON_OUT} ]; then + echo "bootloader not split version" + exit 0 +else + echo "using info from ${FILEINFO_JSON_OUT}" + # find out if the 2 stage bootloader files are available + # files should have stage defined: fileinfo_json files[] -> stage + PRODUCT=$(echo ${PRODUCT}|sed 's/_/-/g') + filename=$(jq '.files[] | select(.hw=="'${PRODUCT}'" and .stage=="ssbl").filename' ${FILEINFO_JSON_OUT}|sed 's/"//g') + + tmp_bl_reponame=$(jq '.reponame' ${FILEINFO_JSON_OUT}) + tmp_bl_sha=$(jq '.sha' ${FILEINFO_JSON_OUT}) + tmp_bl_build_url=$(jq '.build_url' ${FILEINFO_JSON_OUT}) + + # filter files for this product and append $output_dir to .files[].filename + # file path is appended because the files are copied to a different directory so it will match the fileinfo.json + tmp_files=$(jq '.files[] | select(.hw=="'${PRODUCT}'" and .stage=="ssbl") | .filename = "/firmware/'${OUTPUT_DIR}'"+.filename' ${FILEINFO_JSON_OUT}|jq -s .) + + if [ -z "${tmp_files}" ]; then + echo "no ssbl files found" + exit 0 + fi + + tmp_json="{\"${PRODUCT}_bootloader_reponame\":${tmp_bl_reponame},\"${PRODUCT}_bl_sha\":${tmp_bl_sha},\"${PRODUCT}_bl_build_url\":${tmp_bl_build_url},\"bl-files\":${tmp_files}}" + echo $(echo $tmp_json|jq .) + echo $tmp_json|jq . > ${FILEINFO_JSON_OUT} + + # copy each file + echo "make target dir (dirname of) ${OUTPUT_DIR}/${filename}" + mkdir -p $(dirname ${OUTPUT_DIR}/${filename}) + echo "copying ${filename} to ${OUTPUT_DIR}/${filename}" + docker cp ${TMP_CONTAINER_NAME}:${filename} ${OUTPUT_DIR}/${filename} +fi diff --git a/Tools/generate_basic_build_info.sh b/Tools/generate_basic_build_info.sh index e12f4924d69c..254f08e8dcb4 100755 --- a/Tools/generate_basic_build_info.sh +++ b/Tools/generate_basic_build_info.sh @@ -34,6 +34,9 @@ build_info["reponame"]=${repo} build_info["sha"]=${sha} build_info["px4_firmware_sha"]=${px4_sha} build_info["build_url"]=${build_url} +build_info["px4_build_url"]=${build_url} +build_info["px4_reponame"]=${repo} +build_info["px4_sha"]=${sha} # loop thru associative array and print key value pairs json_info="{" diff --git a/Tools/px_uploader.Dockerfile b/Tools/px_uploader.Dockerfile index bf34c98d5ee8..fc1dda5af6a4 100644 --- a/Tools/px_uploader.Dockerfile +++ b/Tools/px_uploader.Dockerfile @@ -5,7 +5,7 @@ ARG SALUKI_FILE_INFO_JSON=saluki_file_info.json ARG PX4_EXPORT_DIR ENV PX4_EXPORT_DIR=$PX4_EXPORT_DIR -COPY $saluki_fpga_directory /firmware/fpga +COPY $saluki_fpga_directory /firmware/combine COPY $SALUKI_FILE_INFO_JSON /$SALUKI_FILE_INFO_JSON WORKDIR /firmware diff --git a/Tools/saluki_packaging b/Tools/saluki_packaging new file mode 160000 index 000000000000..d819118b4019 --- /dev/null +++ b/Tools/saluki_packaging @@ -0,0 +1 @@ +Subproject commit d819118b4019b61508fa7237f7745497ace9ac33 diff --git a/clone_public.sh b/clone_public.sh index 8ad367957616..84e5ba8b64dc 100755 --- a/clone_public.sh +++ b/clone_public.sh @@ -15,6 +15,7 @@ do [[ "${repo}" == *rust_px4_nuttx ]] || \ [[ "${repo}" == *rust_module_example ]] || \ [[ "${repo}" == *secure_udp_proxy ]] || \ + [[ "${repo}" == *saluki_packaging ]] || \ [[ "${repo}" == *process ]] && continue git submodule update --init --recursive "${repo}" done <<< "$(git submodule status | awk '{print $2}')"