From 8297a1352f203288b31245c661f6402261eb68e8 Mon Sep 17 00:00:00 2001 From: hathach Date: Wed, 27 Nov 2024 18:32:09 +0700 Subject: [PATCH 01/10] try to run arm-iar with circleci with new token --- .circleci/config.yml | 15 ++++++++------- .circleci/config2.yml | 19 +++++++++++++++---- .github/workflows/build.yml | 3 +-- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 28afafb006..7c6748b363 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,13 +19,14 @@ jobs: echo "MATRIX_JSON=$MATRIX_JSON" BUILDSYSTEM_TOOLCHAIN=( - "cmake arm-clang" - "make aarch64-gcc" - "make arm-gcc" - "make msp430-gcc" - "make riscv-gcc" - "make rx-gcc" - "cmake esp-idf" + #"cmake arm-clang" + "cmake arm-iar" + #"make aarch64-gcc" + #"make arm-gcc" + #"make msp430-gcc" + #"make riscv-gcc" + #"make rx-gcc" + #"cmake esp-idf" ) RESOURCE_LARGE='["nrf", "imxrt"]' diff --git a/.circleci/config2.yml b/.circleci/config2.yml index e811ef3ede..3d62607f0e 100644 --- a/.circleci/config2.yml +++ b/.circleci/config2.yml @@ -16,7 +16,8 @@ commands: "arm-gcc": "https://github.com/xpack-dev-tools/arm-none-eabi-gcc-xpack/releases/download/v13.2.1-1.1/xpack-arm-none-eabi-gcc-13.2.1-1.1-linux-x64.tar.gz", "msp430-gcc": "http://software-dl.ti.com/msp430/msp430_public_sw/mcu/msp430/MSPGCC/9_2_0_0/export/msp430-gcc-9.2.0.50_linux64.tar.bz2", "riscv-gcc": "https://github.com/xpack-dev-tools/riscv-none-elf-gcc-xpack/releases/download/v13.2.0-2/xpack-riscv-none-elf-gcc-13.2.0-2-linux-x64.tar.gz", - "rx-gcc": "https://llvm-gcc-renesas.com/downloads/get.php?f=rx/8.3.0.202004-gnurx/gcc-8.3.0.202004-GNURX-ELF.run" + "rx-gcc": "https://llvm-gcc-renesas.com/downloads/get.php?f=rx/8.3.0.202004-gnurx/gcc-8.3.0.202004-GNURX-ELF.run", + "arm-iar": "https://updates.iar.com/FileStore/STANDARD/001/003/322/cxarm-9.60.3.deb" }' toolchain_url=$(echo $TOOLCHAIN_JSON | jq -r '.["<< parameters.toolchain >>"]') @@ -38,18 +39,26 @@ commands: # download if folder does not exist (not cached) if [ ! -d ~/cache/<< parameters.toolchain >> ]; then mkdir -p ~/cache/<< parameters.toolchain >> - wget --progress=dot:giga $toolchain_url -O toolchain.tar.gz if [[ << parameters.toolchain >> == rx-gcc ]]; then - mv toolchain.tar.gz toolchain.run + wget --progress=dot:giga $toolchain_url -O toolchain.run chmod +x toolchain.run ./toolchain.run -p ~/cache/<< parameters.toolchain >>/gnurx -y + elif [[ << parameters.toolchain >> == arm-iar ]]; then + wget --progress=dot:giga $toolchain_url -O ~/cache/<< parameters.toolchain >>/toolchain.deb else + wget --progress=dot:giga $toolchain_url -O toolchain.tar.gz tar -C ~/cache/<< parameters.toolchain >> -xaf toolchain.tar.gz fi fi # Add toolchain to PATH - echo "export PATH=$PATH:`echo ~/cache/<< parameters.toolchain >>/*/bin`" >> $BASH_ENV + if [[ << parameters.toolchain >> == arm-iar ]]; then + # Install IAR since we only cache deb file + sudo dpkg --ignore-depends=libusb-1.0-0 -i ~/cache/<< parameters.toolchain >>/toolchain.deb + echo "export PATH=$PATH:/opt/iar/cxarm/arm/bin" >> $BASH_ENV + else + echo "export PATH=$PATH:`echo ~/cache/<< parameters.toolchain >>/*/bin`" >> $BASH_ENV + fi - save_cache: name: Save Toolchain Cache @@ -110,6 +119,8 @@ commands: # Toolchain option default is gcc if [ << parameters.toolchain >> == arm-clang ]; then TOOLCHAIN_OPTION="--toolchain clang" + elif [ << parameters.toolchain >> == arm-iar ]; then + TOOLCHAIN_OPTION="--toolchain iar" elif [ << parameters.toolchain >> == arm-gcc ]; then TOOLCHAIN_OPTION="--toolchain gcc" fi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 283b720616..da565aed88 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -118,8 +118,7 @@ jobs: runs-on: [self-hosted, Linux, X64, hifiphile] env: BUILD_ARGS: ${{ join(fromJSON(needs.set-matrix.outputs.json)['arm-iar'], ' ') }} - IAR_LMS_CLOUD_URL: https://license.cloud.iar.com - IAR_LMS_BEARER_TOKEN: ${{ secrets.IAR_LMS_BEARER_TOKEN }} + # IAR_LMS_BEARER_TOKEN: ${{ secrets.IAR_LMS_BEARER_TOKEN }} steps: - name: Clean workspace run: | From de7d9703c74f0963bae83ab2a567976c22e60326 Mon Sep 17 00:00:00 2001 From: hathach Date: Wed, 27 Nov 2024 19:04:40 +0700 Subject: [PATCH 02/10] print iccarm version --- .circleci/config2.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config2.yml b/.circleci/config2.yml index 3d62607f0e..e6ae87e77f 100644 --- a/.circleci/config2.yml +++ b/.circleci/config2.yml @@ -121,6 +121,7 @@ commands: TOOLCHAIN_OPTION="--toolchain clang" elif [ << parameters.toolchain >> == arm-iar ]; then TOOLCHAIN_OPTION="--toolchain iar" + iccarm --version elif [ << parameters.toolchain >> == arm-gcc ]; then TOOLCHAIN_OPTION="--toolchain gcc" fi From 077824c6731965ff3f0fd464f4cc127aa7aa5c87 Mon Sep 17 00:00:00 2001 From: hathach Date: Wed, 27 Nov 2024 20:03:49 +0700 Subject: [PATCH 03/10] reduce resource class for arm-iar to medium (2 core) --- .circleci/config.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7c6748b363..bdbc284967 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,7 +53,12 @@ jobs: echo " build-system: ['$build_system']" >> .circleci/config2.yml echo " toolchain: ['$toolchain']" >> .circleci/config2.yml echo " family: $FAMILY" >> .circleci/config2.yml - #echo " resource_class: ['medium+']" >> .circleci/config2.yml + # due to limit of IAR capacity, use medium resource + if [[ $toolchain == arm-iar ]]; then + echo " resource_class: ['medium']" >> .circleci/config2.yml + else + echo " resource_class: ['medium+']" >> .circleci/config2.yml + fi # add large resources if [ "$(echo $FAMILY_LARGE | jq 'length')" -gt 0 ]; then From f8a034f400698283afd2444ebda25b56edd24549 Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 29 Nov 2024 11:18:14 +0700 Subject: [PATCH 04/10] change iar docker to large --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bdbc284967..cbc618b611 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,7 +55,7 @@ jobs: echo " family: $FAMILY" >> .circleci/config2.yml # due to limit of IAR capacity, use medium resource if [[ $toolchain == arm-iar ]]; then - echo " resource_class: ['medium']" >> .circleci/config2.yml + echo " resource_class: ['large']" >> .circleci/config2.yml else echo " resource_class: ['medium+']" >> .circleci/config2.yml fi From 99cc37744ff77553101635cdbc44257b4f5f9667 Mon Sep 17 00:00:00 2001 From: hathach Date: Thu, 5 Dec 2024 12:08:32 +0700 Subject: [PATCH 05/10] limit iar ci parallel build to 4 --- tools/build.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/build.py b/tools/build.py index 3a9239bc26..7272d57717 100755 --- a/tools/build.py +++ b/tools/build.py @@ -109,7 +109,11 @@ def cmake_board(board, toolchain, build_flags_on): rcmd = run_cmd(f'cmake examples -B {build_dir} -G "Ninja" -DBOARD={board} -DCMAKE_BUILD_TYPE=MinSizeRel ' f'-DTOOLCHAIN={toolchain} {build_flags}') if rcmd.returncode == 0: - rcmd = run_cmd(f"cmake --build {build_dir}") + cmd = f"cmake --build {build_dir}" + # Due to IAR capability, limit parallel build to 4 + if toolchain == 'iar' and os.getenv('CIRCLECI'): + cmd += ' --parallel 4' + rcmd = run_cmd(cmd) ret[0 if rcmd.returncode == 0 else 1] += 1 example = 'all' From b897b7b65ea815bbcfeaf7b138b2db41f9bdf08b Mon Sep 17 00:00:00 2001 From: hathach Date: Thu, 5 Dec 2024 12:33:49 +0700 Subject: [PATCH 06/10] use docker medium+ --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cbc618b611..26bcc601dc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,11 +54,11 @@ jobs: echo " toolchain: ['$toolchain']" >> .circleci/config2.yml echo " family: $FAMILY" >> .circleci/config2.yml # due to limit of IAR capacity, use medium resource - if [[ $toolchain == arm-iar ]]; then - echo " resource_class: ['large']" >> .circleci/config2.yml - else - echo " resource_class: ['medium+']" >> .circleci/config2.yml - fi + #if [[ $toolchain == arm-iar ]]; then + # echo " resource_class: ['large']" >> .circleci/config2.yml + #else + echo " resource_class: ['medium+']" >> .circleci/config2.yml + #fi # add large resources if [ "$(echo $FAMILY_LARGE | jq 'length')" -gt 0 ]; then From 463d6e8659309194ea95a249b5e92e65bfcaa52d Mon Sep 17 00:00:00 2001 From: hathach Date: Thu, 5 Dec 2024 13:33:13 +0700 Subject: [PATCH 07/10] add hil-hfp to compile and test with IAR tweak IAR ci --- .circleci/config.yml | 7 +----- .github/workflows/build.yml | 7 +----- .github/workflows/ci_set_matrix.py | 8 ------ .github/workflows/hil_test.yml | 39 +++++++++++++++++++++++++++++- 4 files changed, 40 insertions(+), 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 26bcc601dc..0f7b2e9cdd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,7 +29,7 @@ jobs: #"cmake esp-idf" ) - RESOURCE_LARGE='["nrf", "imxrt"]' + RESOURCE_LARGE='["nrf", "imxrt", "stm32f4", "stm32h7"]' for e in "${BUILDSYSTEM_TOOLCHAIN[@]}"; do e_arr=($e) @@ -53,12 +53,7 @@ jobs: echo " build-system: ['$build_system']" >> .circleci/config2.yml echo " toolchain: ['$toolchain']" >> .circleci/config2.yml echo " family: $FAMILY" >> .circleci/config2.yml - # due to limit of IAR capacity, use medium resource - #if [[ $toolchain == arm-iar ]]; then - # echo " resource_class: ['large']" >> .circleci/config2.yml - #else echo " resource_class: ['medium+']" >> .circleci/config2.yml - #fi # add large resources if [ "$(echo $FAMILY_LARGE | jq 'length')" -gt 0 ]; then diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index da565aed88..67482c748b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -113,7 +113,7 @@ jobs: # Build IAR on HFP self-hosted # --------------------------------------- arm-iar: - if: github.repository_owner == 'hathach' + if: github.repository_owner == 'hathach' && github.event_name == 'push' needs: set-matrix runs-on: [self-hosted, Linux, X64, hifiphile] env: @@ -137,8 +137,3 @@ jobs: - name: Build run: python3 tools/build.py --one-per-family --toolchain iar $BUILD_ARGS - - - name: Test on actual hardware (hardware in the loop) - if: github.event_name == 'pull_request' - run: | - python3 test/hil/hil_test.py hfp.json diff --git a/.github/workflows/ci_set_matrix.py b/.github/workflows/ci_set_matrix.py index 705dcac94c..394ca4be90 100755 --- a/.github/workflows/ci_set_matrix.py +++ b/.github/workflows/ci_set_matrix.py @@ -55,14 +55,6 @@ def set_matrix_json(): for toolchain in toolchain_list: filtered_families = [family for family, supported_toolchain in family_list.items() if toolchain in supported_toolchain] - - # always add board in hfp.json for arm-iar - if toolchain == 'arm-iar': - with open('test/hil/hfp.json') as f: - hfp_data = json.load(f) - hfp_boards = [f"-b{board['name']}" for board in hfp_data['boards']] - filtered_families = filtered_families + hfp_boards - matrix[toolchain] = filtered_families print(json.dumps(matrix)) diff --git a/.github/workflows/hil_test.yml b/.github/workflows/hil_test.yml index c6e7f33b0b..22fea77bed 100644 --- a/.github/workflows/hil_test.yml +++ b/.github/workflows/hil_test.yml @@ -58,7 +58,7 @@ jobs: # --------------------------------------- # Hardware in the loop (HIL) - # self-hosted running on an VM. For attached hardware checkout HIL_JSON + # self-hosted on local VM, for attached hardware checkout HIL_JSON # --------------------------------------- hil-tinyusb: if: github.repository_owner == 'hathach' @@ -86,3 +86,40 @@ jobs: run: | ls cmake-build/ python3 test/hil/hil_test.py ${{ env.HIL_JSON }} + + # --------------------------------------- + # Hardware in the loop (HIL) + # self-hosted by HFP, build with IAR toolchain, for attached hardware checkout test/hil/hfp.json + # --------------------------------------- + hil-hfp: + if: github.repository_owner == 'hathach' + runs-on: [self-hosted, Linux, X64, hifiphile] + #env: + # IAR_LMS_BEARER_TOKEN: ${{ secrets.IAR_LMS_BEARER_TOKEN }} + steps: + - name: Clean workspace + run: | + echo "Cleaning up previous run" + rm -rf "${{ github.workspace }}" + mkdir -p "${{ github.workspace }}" + + - name: Toolchain version + run: iccarm --version + + - name: Checkout TinyUSB + uses: actions/checkout@v4 + + - name: Get build boards + run: | + MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py test/hil/hfp.json) + BUILD_ARGS=$(echo $MATRIX_JSON | jq -r '.["arm-gcc"] | join(" ")') + echo "BUILD_ARGS=$BUILD_ARGS" >> $GITHUB_ENV + + - name: Get Dependencies + run: python3 tools/get_deps.py $BUILD_ARGS + + - name: Build + run: python3 tools/build.py --toolchain iar $BUILD_ARGS + + - name: Test on actual hardware (hardware in the loop) + run: python3 test/hil/hil_test.py hfp.json From 39995c59d40d79748a3cd509958ba1ec8cfe7614 Mon Sep 17 00:00:00 2001 From: hathach Date: Thu, 5 Dec 2024 15:05:05 +0700 Subject: [PATCH 08/10] wrap up --- .circleci/config.yml | 14 +++++++------- .github/workflows/hil_test.yml | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0f7b2e9cdd..4db1a5be5b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,14 +19,14 @@ jobs: echo "MATRIX_JSON=$MATRIX_JSON" BUILDSYSTEM_TOOLCHAIN=( - #"cmake arm-clang" + "cmake arm-clang" "cmake arm-iar" - #"make aarch64-gcc" - #"make arm-gcc" - #"make msp430-gcc" - #"make riscv-gcc" - #"make rx-gcc" - #"cmake esp-idf" + "make aarch64-gcc" + "make arm-gcc" + "make msp430-gcc" + "make riscv-gcc" + "make rx-gcc" + "cmake esp-idf" ) RESOURCE_LARGE='["nrf", "imxrt", "stm32f4", "stm32h7"]' diff --git a/.github/workflows/hil_test.yml b/.github/workflows/hil_test.yml index 22fea77bed..bfa2b57047 100644 --- a/.github/workflows/hil_test.yml +++ b/.github/workflows/hil_test.yml @@ -113,6 +113,7 @@ jobs: run: | MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py test/hil/hfp.json) BUILD_ARGS=$(echo $MATRIX_JSON | jq -r '.["arm-gcc"] | join(" ")') + echo "BUILD_ARGS=$BUILD_ARGS" echo "BUILD_ARGS=$BUILD_ARGS" >> $GITHUB_ENV - name: Get Dependencies From a4432fd48f81a616456ca0588c0c3ae4a6b0d25c Mon Sep 17 00:00:00 2001 From: hathach Date: Thu, 5 Dec 2024 16:27:05 +0700 Subject: [PATCH 09/10] clean up ci --- .circleci/config.yml | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4db1a5be5b..451b790854 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,6 +31,26 @@ jobs: RESOURCE_LARGE='["nrf", "imxrt", "stm32f4", "stm32h7"]' + gen_build_entry() { + local build_system="$1" + local toolchain="$2" + local family="$3" + local resource_class="$4" + + if [[ "$toolchain" == "esp-idf" ]]; then + echo " - build-vm:" >> .circleci/config2.yml + else + echo " - build:" >> .circleci/config2.yml + fi + + echo " matrix:" >> .circleci/config2.yml + echo " parameters:" >> .circleci/config2.yml + echo " build-system: ['$build_system']" >> .circleci/config2.yml + echo " toolchain: ['$toolchain']" >> .circleci/config2.yml + echo " family: $family" >> .circleci/config2.yml + echo " resource_class: ['$resource_class']" >> .circleci/config2.yml + } + for e in "${BUILDSYSTEM_TOOLCHAIN[@]}"; do e_arr=($e) build_system="${e_arr[0]}" @@ -44,26 +64,14 @@ jobs: FAMILY=$(jq -n --argjson family "$FAMILY" --argjson resource "$RESOURCE_LARGE" '$family | map(select(IN($resource[]) | not))') if [[ $toolchain == esp-idf ]]; then - echo " - build-vm:" >> .circleci/config2.yml + gen_build_entry "$build_system" "$toolchain" "$FAMILY" "large" else - echo " - build:" >> .circleci/config2.yml - fi - echo " matrix:" >> .circleci/config2.yml - echo " parameters:" >> .circleci/config2.yml - echo " build-system: ['$build_system']" >> .circleci/config2.yml - echo " toolchain: ['$toolchain']" >> .circleci/config2.yml - echo " family: $FAMILY" >> .circleci/config2.yml - echo " resource_class: ['medium+']" >> .circleci/config2.yml + gen_build_entry "$build_system" "$toolchain" "$FAMILY" "medium+" - # add large resources - if [ "$(echo $FAMILY_LARGE | jq 'length')" -gt 0 ]; then - echo " - build:" >> .circleci/config2.yml - echo " matrix:" >> .circleci/config2.yml - echo " parameters:" >> .circleci/config2.yml - echo " build-system: ['$build_system']" >> .circleci/config2.yml - echo " toolchain: ['$toolchain']" >> .circleci/config2.yml - echo " family: $FAMILY_LARGE" >> .circleci/config2.yml - echo " resource_class: ['large']" >> .circleci/config2.yml + # add large resources if available + if [ "$(echo $FAMILY_LARGE | jq 'length')" -gt 0 ]; then + gen_build_entry "$build_system" "$toolchain" "$FAMILY_LARGE" "large" + fi fi done From 9084bbd305c0f36bd80347bea964d36298e0fd83 Mon Sep 17 00:00:00 2001 From: hathach Date: Thu, 5 Dec 2024 16:56:28 +0700 Subject: [PATCH 10/10] increase iar parallel for docker large --- tools/build.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/build.py b/tools/build.py index 7272d57717..bbc98e9c48 100755 --- a/tools/build.py +++ b/tools/build.py @@ -110,9 +110,12 @@ def cmake_board(board, toolchain, build_flags_on): f'-DTOOLCHAIN={toolchain} {build_flags}') if rcmd.returncode == 0: cmd = f"cmake --build {build_dir}" - # Due to IAR capability, limit parallel build to 4 + # Due to IAR capability, limit parallel build to 4 (medium+) or 6 (large) docker if toolchain == 'iar' and os.getenv('CIRCLECI'): - cmd += ' --parallel 4' + if 'large' in os.getenv('CIRCLE_JOB'): + cmd += ' --parallel 6' + else: + cmd += ' --parallel 4' rcmd = run_cmd(cmd) ret[0 if rcmd.returncode == 0 else 1] += 1