Skip to content

CI

CI #9681

Workflow file for this run

name: CI
on:
schedule:
- cron: '00 2 * * *' # 02:00 UTC
push:
branches-ignore:
- "wip/**"
- "wip-**"
- "*/wip-**"
- "*/wip/**"
release:
types: [published]
workflow_dispatch:
inputs:
# Use 'inputs.<name>' when pasting into a bash `if`, like so:
# if [[ "${{ inputs.strip }}" == 'true' ]]; then ...
#
# When using to guard a step, use 'github.event.inputs.<name>', like so:
# if: ${{ github.event.inputs.<name> == 'true' }}
# or
# if: ${{ github.event.inputs.<name> != 'false' }}
# In the first case, the step will not run if there isn't any workflow dispatch.
# In the second case, the step will run if there isn't any workflow dispatch.
# As such, the recommendation is to use the `== true` if the default is
# false, and the `!= false` if the default is true.
strip:
description: "Strip executables"
required: false
type: boolean
default: false
sign-macos:
description: "Sign macOS executables"
required: false
type: boolean
default: false
build-arm-macos:
description: "Build ARM macOS executables"
required: false
type: boolean
default: false
dispatch-toitdocs:
description: "Dispatch toitdoc rebuild"
required: false
type: boolean
default: false
run-tests:
description: "Run tests"
required: false
type: boolean
default: true
test-more:
description: "Run more tests"
required: false
type: boolean
default: false
run-esp32-flow:
description: "Run the ESP32 flow"
required: false
type: boolean
default: true
do-release:
description: "Build release artifacts and upload with this version"
required: false
type: string
default:
env:
TOIT_VERSION: ${{ github.event.inputs.do-release || github.event.release.tag_name || '' }}
DO_RELEASE: ${{ github.event_name == 'release' || startsWith(github.event.inputs.do-release, 'v') }}
DO_MORE_TESTS: ${{ github.event_name == 'schedule' || github.event.inputs.test-more == 'true' }}
ESPTOOL_VERSION: "v4.8.1%2Btoitlang"
jobs:
prereqs:
runs-on: ubuntu-latest
steps:
- name: Set strategy variables
id: vars
run: |
# ubuntu-20.04 so that we don't depend on a recent glibc.
# macos-13, since it's still intel based.
if [[ "${{ github.ref }}" == 'refs/heads/master' || \
"${{ env.DO_RELEASE }}" == "true" || \
"${{ inputs.build-arm-macos }}" == "true" || \
"${{ inputs.sign-macos }}" == "true" \
]]; then
echo 'CONTAINERS=[ "ubuntu-20.04", "macos-13", "macos-latest", "windows-latest" ]' > $GITHUB_OUTPUT
else
echo 'CONTAINERS=[ "ubuntu-20.04", "macos-13", "windows-latest" ]' > $GITHUB_OUTPUT
fi
if [[ "${{ inputs.run-tests }}" != "false" ]]; then
echo 'SHARDS=[1, 2, 3, 4, 5]' >> $GITHUB_OUTPUT
echo 'EXTRA_INCLUDES=[{"container": "ubuntu-20.04", "shard": 6}, {"container": "ubuntu-20.04", "shard": 7}, {"container": "ubuntu-20.04", "shard": 8}]' >> $GITHUB_OUTPUT
echo 'SHARD_COUNT=5' >> $GITHUB_OUTPUT
echo 'SHARD_COUNT_LINUX=8' >> $GITHUB_OUTPUT
else
echo 'SHARDS=[1]' >> $GITHUB_OUTPUT
echo 'EXTRA_INCLUDES=[]' >> $GITHUB_OUTPUT
echo 'SHARD_COUNT=1' >> $GITHUB_OUTPUT
fi
outputs:
CONTAINERS: ${{ steps.vars.outputs.CONTAINERS }}
SHARDS: ${{ steps.vars.outputs.SHARDS }}
EXTRA_INCLUDES: ${{ steps.vars.outputs.EXTRA_INCLUDES }}
SHARD_COUNT: ${{ steps.vars.outputs.SHARD_COUNT }}
SHARD_COUNT_LINUX: ${{ steps.vars.outputs.SHARD_COUNT_LINUX }}
build:
needs: prereqs
strategy:
# The matrix will run all possible combinations in parallel.
matrix:
container: ${{ fromJson(needs.prereqs.outputs.CONTAINERS) }}
shard: ${{ fromJson(needs.prereqs.outputs.SHARDS) }}
include: ${{ fromJson(needs.prereqs.outputs.EXTRA_INCLUDES) }}
runs-on: ${{ matrix.container }}
steps:
- name: Show inputs
if: github.event_name == 'workflow_dispatch'
run: echo "${{ toJSON(github.event.inputs) }}"
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up constants
id: constants
shell: bash
run: |
echo "flaky_result=flaky-result/${{ matrix.container }}-${{ matrix.shard }}.txt" >> $GITHUB_OUTPUT
echo "flaky_result_name=flaky-result-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.container }}-${{ matrix.shard }}" >> $GITHUB_OUTPUT
echo "external_result=external-result/${{ matrix.container }}-${{ matrix.shard }}.txt" >> $GITHUB_OUTPUT
echo "external_result_name=external-result-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.container }}-${{ matrix.shard }}" >> $GITHUB_OUTPUT
echo "debug_root_dir=debug-ci" >> $GITHUB_OUTPUT
echo "debug_dir=debug-ci/$RUNNER_OS-${{ matrix.shard }}" >> $GITHUB_OUTPUT
if [ -z "$TOIT_VERSION" ]; then
TOIT_VERSION=$(cmake -DPRINT_VERSION=1 -P tools/gitversion.cmake)
fi
echo $TOIT_VERSION
echo "toit_version=$TOIT_VERSION" >> $GITHUB_OUTPUT
echo "TOIT_GIT_VERSION=$TOIT_VERSION" >> $GITHUB_ENV
if [[ "$RUNNER_OS" == "Linux" ]]; then
echo "artifact=toit-linux.tar.gz" >> $GITHUB_OUTPUT
echo "total_shards=${{ needs.prereqs.outputs.SHARD_COUNT_LINUX }}" >> $GITHUB_OUTPUT
elif [[ "$RUNNER_OS" == "macOS" ]] && [[ "$RUNNER_ARCH" == "X64" ]]; then
echo "artifact=toit-macos.tar.gz" >> $GITHUB_OUTPUT
echo "total_shards=${{ needs.prereqs.outputs.SHARD_COUNT }}" >> $GITHUB_OUTPUT
elif [[ "$RUNNER_OS" == "macOS" ]] && [[ "$RUNNER_ARCH" == "ARM64" ]]; then
echo "artifact=toit-macos-arm.tar.gz" >> $GITHUB_OUTPUT
echo "total_shards=${{ needs.prereqs.outputs.SHARD_COUNT }}" >> $GITHUB_OUTPUT
elif [[ "$RUNNER_OS" == "Windows" ]]; then
echo "artifact=toit-windows.tar.gz" >> $GITHUB_OUTPUT
echo "total_shards=${{ needs.prereqs.outputs.SHARD_COUNT }}" >> $GITHUB_OUTPUT
else
echo "UNSUPPORTED RUNNER: $RUNNER_OS"
exit 1
fi
# Anything in the debug dir will be uploaded as artifact.
- name: Setup debug dir
shell: sh
run: |
mkdir -p ${{ steps.constants.outputs.debug_dir }}
- name: Setup build environment
uses: ./actions/setup-build
with:
toit-dir: .
- name: Setup compilation env
shell: bash
run: |
echo "SHARD_START=${{ matrix.shard }}" >> $GITHUB_ENV
echo "SHARD_STRIDE=${{ steps.constants.outputs.total_shards }}" >> $GITHUB_ENV
- name: Ccache stats
run: ccache -s
# Build using make.
- name: Build sdk
shell: bash
# Note that the TOIT_GIT_VERSION is set. The make step will thus not
# compute the version from git.
run: |
make sdk
- name: Ccache stats
run: ccache -s
- name: Build test SDKs - Linux only
if: env.DO_MORE_TESTS == 'true' && runner.os == 'Linux'
run: |
make HOST=host32 BUILD_TYPE=Debug sdk
make HOST=host-ctp TOOLCHAIN=host TOIT_CHECK_PROPAGATED_TYPES=1 sdk
- name: Build test SDKs
shell: bash
if: env.DO_MORE_TESTS == 'true'
run: |
make BUILD=build-debug BUILD_TYPE=Debug sdk
# It's wasteful to rebuild the SDK just to change the optimization level and assert, but
# it's the easiest way to make sure all tests are run in that configuration.
TOIT_OPTIMIZATION_OVERRIDE=0 TOIT_ASSERT_OVERRIDE=0 make HOST=host-O0 TOOLCHAIN=host sdk
TOIT_OPTIMIZATION_OVERRIDE=2 TOIT_ASSERT_OVERRIDE=1 make HOST=host-O2 TOOLCHAIN=host sdk
- name: Ccache stats
run: ccache -s
- name: Create ccache debug file
if: runner.os == 'Linux' && matrix.shard == 1
# Create the preprocessed file for the primitive_events.cc file.
# This file was chose arbitrarily.
run: |
mkdir ccache-debug
cd build/host
/usr/bin/c++ -fdebug-prefix-map=/home/runner/work/toit/toit=. -Wall -ffunction-sections -fdata-sections -fno-exceptions -Os -fno-rtti -std=gnu++11 -fdiagnostics-color=always -I../../include -I../../third_party/esp-idf/components/mbedtls/mbedtls/include -DMBEDTLS_SSL_IN_CONTENT_LEN=7800 -DMBEDTLS_SSL_OUT_CONTENT_LEN=3700 -DMBEDTLS_PLATFORM_MEMORY=1 -DMBEDTLS_X509_TRUSTED_CERTIFICATE_CALLBACK=1 -DTOIT_DEPLOY -MD -MT src/CMakeFiles/toit_core.dir/primitive_events.cc.o -MF src/CMakeFiles/toit_core.dir/primitive_events.cc.o.d -E ../../src/primitive_events.cc > ../../ccache-debug/primitive_events.cc.i
- name: Upload ccache debug file
if: runner.os == 'Linux' && matrix.shard == 1
uses: actions/upload-artifact@v4
with:
path: ccache-debug
name: ccache-debug
- name: Strip executables - Linux
if: (runner.os == 'Linux' && (env.DO_RELEASE == 'true' || github.event.inputs.strip == 'true'))
# Note: Go executables may not be stripped on Windows.
run: |
strip -s build/host/sdk/bin/*
# Test.
- name: Test
if: ${{ github.event.inputs.run-tests != 'false' }}
shell: bash # This is crucial, as the powershell doesn't abort when there is an error.
run: |
make test
make test-health
- name: Test flaky
if: ${{ github.event.inputs.run-tests != 'false' }}
continue-on-error: true
shell: bash # This is crucial, as the powershell doesn't abort when there is an error.
run: |
mkdir -p $(dirname ${{ steps.constants.outputs.flaky_result }})
echo fail > ${{ steps.constants.outputs.flaky_result }}
make test-flaky
# If we reach here, then the test-flaky target succeeded.
echo "SUCCESS"
echo success > ${{ steps.constants.outputs.flaky_result }}
- name: Test type propagator
if: env.DO_MORE_TESTS == 'true' && runner.os == 'Linux'
run: |
make HOST=host-ctp TOIT_CHECK_PROPAGATED_TYPES=1 test
- name: Test 32-bit debug
if: env.DO_MORE_TESTS == 'true' && runner.os == 'Linux'
run: |
make HOST=host32 BUILD_TYPE=Debug test
- name: Test 64-bit debug
shell: bash
# TODO(florian): fix Windows.
if: env.DO_MORE_TESTS == 'true' && runner.os != 'Windows'
run: |
make BUILD=build-debug BUILD_TYPE=Debug test
- name: Test -O0 no asserts
shell: bash
# TODO(florian): Windows is currently too flaky with -O0.
if: env.DO_MORE_TESTS == 'true' && runner.os != 'Windows'
run: |
TOIT_OPTIMIZATION_OVERRIDE=0 TOIT_ASSERT_OVERRIDE=0 make HOST=host-O0 TOOLCHAIN=host test
- name: Test -O2 with asserts
shell: bash
# TODO(florian): fix Windows.
if: env.DO_MORE_TESTS == 'true' && runner.os != 'Windows'
run: |
TOIT_OPTIMIZATION_OVERRIDE=2 TOIT_ASSERT_OVERRIDE=1 make HOST=host-O2 TOOLCHAIN=host test
- name: Test install
if : matrix.shard == 1
shell: bash
run: |
make prefix=$PWD/build/install-out install
build/install-out/bin/toit.run --version
build/install-out/bin/toit.run -s "print 499"
# Test external code.
- name: Test external
if: ${{ github.event.inputs.run-tests != 'false' }}
# External tests are allowed to fail, but we record them and
# fail a non-required job if they do.
continue-on-error: true
env:
IDF_PATH: ${{ github.workspace }}/third_party/esp-idf
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_COMPRESS: true
CCACHE_COMPRESSLEVEL: '6'
CCACHE_MAXSIZE: 400M
SHARD_START: ${{ matrix.shard }}
SHARD_STRIDE: ${{ steps.constants.outputs.total_shards }}
shell: bash
run: |
# Mark the external test as failing, and update it at the end of this step.
mkdir -p $(dirname ${{ steps.constants.outputs.external_result }})
echo fail > ${{ steps.constants.outputs.external_result }}
make enable-external
find external > ${{ steps.constants.outputs.debug_dir }}/${{ matrix.container }}_${{ matrix.shard }}_files_after_enable_external.log
make test-external
make test-external-health
if [[ "$DO_MORE_TESTS" == "true" ]] && [[ "$RUNNER_OS" == "Linux" ]]; then
cmake -DTOIT_TEST_EXTERNAL=ON build/host32
make HOST=host32 BUILD_TYPE=Debug rebuild-cmake
cd build/host32
ninja check_external
cd ../..
fi
# If we reach here, all steps of this step succeeded.
echo "SUCCESS"
echo success > ${{ steps.constants.outputs.external_result }}
- name: Store flaky test result
uses: actions/upload-artifact@v4
with:
name: ${{ steps.constants.outputs.flaky_result_name }}
path: ${{ steps.constants.outputs.flaky_result }}
- name: Store external test result
uses: actions/upload-artifact@v4
with:
name: ${{ steps.constants.outputs.external_result_name }}
path: ${{ steps.constants.outputs.external_result }}
- name: Toitdoc - Linux
if: runner.os == 'Linux' && matrix.shard == 1
env:
TOIT_VERSION: ${{ steps.constants.outputs.toit_version }}
TOIT_BIN: ${{ github.workspace }}/build/host/sdk/bin/toit
SDKDIR: ${{ github.workspace }}/build/host/sdk
shell: bash
run: |
JSON_OUT_PATH=${{ github.workspace }}/toitdoc_$TOIT_VERSION.json
cd "$SDKDIR"
"$TOIT_BIN" doc build --sdk "--sdk-dir=$SDKDIR" "--output=$JSON_OUT_PATH" --version=$TOIT_VERSION
- name: Pack artifacts
if: matrix.shard == 1
shell: bash
# Note that we use `cp -R` first, since that works on every platform.
run: |
cp -R ./build/host/sdk ./build/host/toit
tar -czf build/${{ steps.constants.outputs.artifact }} -C ./build/host --dereference toit
- name: Prepare debug repros
if: always()
continue-on-error: true
shell: bash
run: |
ls /tmp/lsp_repro-* || true
REPRO_DIR=${{ steps.constants.outputs.debug_dir }}/repros
mkdir -p $REPRO_DIR
find /tmp -maxdepth 1 -name 'lsp_repro-*' -exec cp -r {} $REPRO_DIR \;
ls $REPRO_DIR
- name: Upload debug artifacts
if: always()
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: debug-${{ matrix.container }}-${{ matrix.shard }}
path: ${{ steps.constants.outputs.debug_root_dir }}
retention-days: 5
if-no-files-found: ignore
- name: Upload toitdoc - Linux
if: runner.os == 'Linux' && matrix.shard == 1
uses: actions/upload-artifact@v4
with:
name: toitdoc
path: toitdoc_${{ steps.constants.outputs.toit_version }}.json
- name: Upload toitdoc release artifact - Linux
if: |
runner.os == 'Linux' && matrix.shard == 1 &&
env.DO_RELEASE == 'true' && github.repository_owner == 'toitlang'
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: toitdoc_${{ steps.constants.outputs.toit_version }}.json
asset_name: toitdoc.json
tag: ${{ env.TOIT_VERSION }}
overwrite: true
- name: Dispatch toitdoc-dev rebuild - Linux
if: |
runner.os == 'Linux' && matrix.shard == 1 &&
github.ref == 'refs/heads/master' &&
github.repository_owner == 'toitlang'
uses: actions/github-script@v7
with:
github-token: ${{ secrets.LEON_REPOSITORY_DISPATCH }}
script: |
await github.rest.actions.createWorkflowDispatch({
owner: 'toitware',
repo: 'web-toitdocs',
workflow_id: 'ci.yml',
ref: 'master',
inputs: {
'deploy': 'false',
'toit-version': '',
'toitlang-run-id': '${{ github.run_id }}',
},
});
- name: Dispatch toitdoc rebuild - Linux
if: |
runner.os == 'Linux' && matrix.shard == 1 &&
((env.DO_RELEASE == 'true' && github.repository_owner == 'toitlang') ||
github.event.inputs.dispatch-toitdocs == 'true')
uses: actions/github-script@v7
with:
github-token: ${{ secrets.LEON_REPOSITORY_DISPATCH }}
script: |
await github.rest.actions.createWorkflowDispatch({
owner: 'toitware',
repo: 'web-toitdocs',
workflow_id: 'ci.yml',
ref: 'master',
inputs: {
'deploy': 'true',
'toit-version': '${{ env.TOIT_VERSION }}'
},
});
- name: Upload unsigned artifacts - Windows
if: runner.os == 'Windows' && matrix.shard == 1
uses: actions/upload-artifact@v4
with:
name: ${{ runner.os }}-build-unsigned
path: build/${{ steps.constants.outputs.artifact }}
- name: Upload unsigned artifacts - macOS
if: runner.os == 'macOS' && matrix.shard == 1
uses: actions/upload-artifact@v4
with:
name: ${{ runner.os }}-build-unsigned-${{ matrix.container }}
path: build/${{ steps.constants.outputs.artifact }}
- name: Upload artifacts - Linux
if: runner.os == 'Linux' && matrix.shard == 1
uses: actions/upload-artifact@v4
with:
name: ${{ runner.os }}-build
path: build/${{ steps.constants.outputs.artifact }}
esp32:
if: ${{ github.event.inputs.run-esp32-flow != 'false' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up build environment
uses: ./actions/setup-build
with:
toit-dir: .
esp32: true
cache-key-prefix: esp32
# Build using make.
- name: make
run: |
ccache -s
source third_party/esp-idf/export.sh
# Install the Python packages necessary for MbedTLS after
# setting up the virtual Python environment.
pip install jsonschema jinja2
make esp32
make ESP32_CHIP=esp32c3 esp32
make ESP32_CHIP=esp32s2 esp32
make ESP32_CHIP=esp32s3 esp32
ccache -s
- name: Pack firmware artifacts
shell: bash
run: |
gzip --to-stdout build/esp32/firmware.envelope > build/firmware-esp32.gz
gzip --to-stdout build/esp32c3/firmware.envelope > build/firmware-esp32c3.gz
gzip --to-stdout build/esp32s2/firmware.envelope > build/firmware-esp32s2.gz
gzip --to-stdout build/esp32s3/firmware.envelope > build/firmware-esp32s3.gz
- name: Upload firmware artifacts
uses: actions/upload-artifact@v4
with:
name: firmwares
path: |
build/firmware-esp32.gz
build/firmware-esp32c3.gz
build/firmware-esp32s2.gz
build/firmware-esp32s3.gz
cross:
runs-on: ubuntu-latest
steps:
- name: Set up constants
id: constants
shell: bash
run: |
echo raspberry_pi_artifact=toit-rpi.tar.gz >> $GITHUB_OUTPUT
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set version
run: |
if [ -z "$TOIT_VERSION" ]; then
TOIT_VERSION=$(cmake -DPRINT_VERSION=1 -P tools/gitversion.cmake)
fi
echo "TOIT_GIT_VERSION=$TOIT_VERSION" >> $GITHUB_ENV
- name: Set up build environment
uses: ./actions/setup-build
with:
toit-dir: .
cache-key-prefix: cross
- name: Install mingw
run: |
sudo apt-get install mingw-w64
- name: Make envelopes
run: |
make TARGET=win64 build-envelope
make TARGET=raspbian build-envelope
make TARGET=aarch64 build-envelope
make TARGET=arm-linux-gnueabi build-envelope
- name: Check envelopes exist
run: |
ls build/win64/firmware.envelope
ls build/raspbian/firmware.envelope
ls build/aarch64/firmware.envelope
ls build/arm-linux-gnueabi/firmware.envelope
- name: Make
# Note that the TOIT_GIT_VERSION is set. The make step will thus not
# compute the version from git.
run: |
make sdk
make TARGET=win64 sdk
make raspbian
make aarch64
make arm-linux-gnueabi
- name: Build test snapshot
run: |
echo '
import .system.extensions.host.run-image-exit-codes
main:
print "hello world"
exit EXIT-CODE-STOP
' > hello.toit
build/host/sdk/bin/toit compile --snapshot -o hello.snapshot hello.toit
- name: Customize and extract envelopes
run: |
for arch in raspbian aarch64; do
cp build/$arch/firmware.envelope $arch.envelope
build/host/sdk/bin/toit tool firmware container install \
hello hello.snapshot \
-e $arch.envelope
build/host/sdk/bin/toit tool firmware extract \
-e $arch.envelope \
--format "tar" \
-o build/$arch/extracted.tar
mkdir build/$arch/extracted
tar -xf build/$arch/extracted.tar -C build/$arch/extracted
done
- name: Test executables 32-bit
uses: pguyot/arm-runner-action@v2
with:
cpu: cortex-a7
base_image: "raspios_lite:2022-01-28"
bind_mount_repository: true
commands: |
uname -m > pi_uname
build/raspbian/sdk/bin/toit.pkg version
build/raspbian/sdk/bin/toit.run --version
build/raspbian/sdk/tools/firmware --help
build/raspbian/extracted/boot.sh > raspbian_output
- name: Check that the 32-bit run-script worked
run: |
cat raspbian_output
grep "hello world" raspbian_output
- name: Test executables 64-bit
uses: pguyot/arm-runner-action@v2
with:
cpu: cortex-a7
base_image: "raspios_lite_arm64:2022-01-28"
bind_mount_repository: true
commands: |
uname -m > pi_uname
build/aarch64/sdk/bin/toit.pkg version
build/aarch64/sdk/bin/toit.run --version
build/aarch64/sdk/tools/firmware --help
build/aarch64/extracted/boot.sh > aarch64_output
- name: Check that the 64-bit run-script worked
run: |
cat aarch64_output
grep "hello world" aarch64_output
- name: Pack artifacts
shell: bash
# Note that we use `cp -R` first, since that works on every platform.
run: |
cp -R ./build/raspbian/sdk ./build/raspbian/toit
tar -czf build/${{ steps.constants.outputs.raspberry_pi_artifact }} -C ./build/raspbian --dereference toit
cp -R ./build/aarch64/sdk ./build/aarch64/toit
tar -czf build/toit-aarch64.tar.gz -C ./build/aarch64 --dereference toit
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: ${{ runner.os }}-build-cross
path: |
build/${{ steps.constants.outputs.raspberry_pi_artifact }}
build/toit-aarch64.tar.gz
combine:
needs: [build, cross]
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v4
with:
name: Windows-build-unsigned
- uses: actions/download-artifact@v4
with:
pattern: Linux-build*
merge-multiple: true
- uses: actions/download-artifact@v4
with:
pattern: macOS-build-unsigned*
merge-multiple: true
- name: Set up macOS ARM environment
run: |
if [[ "${{ env.DO_RELEASE }}" == 'true' || \
"${{ inputs.build-arm-macos }}" == 'true' || \
"${{ github.ref }}" == 'refs/heads/master' \
]]; then
echo "MACOS_ARM=macos-arm" >> $GITHUB_ENV;
EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
echo "MACOS_TARS<<$EOF" >> $GITHUB_ENV;
echo "toit-macos.tar.gz" >> $GITHUB_ENV;
echo "toit-macos-arm.tar.gz" >> $GITHUB_ENV;
echo "$EOF" >> $GITHUB_ENV;
else
echo "MACOS_TARS=toit-macos.tar.gz" >> $GITHUB_ENV;
fi
- name: Unpack SDKs
run: |
for os in windows linux macos rpi aarch64 $MACOS_ARM; do
mkdir -p $os
tar x -zf toit-$os.tar.gz -C $os
done
- name: Copy vessels
run: |
mkdir -p vessels
for source in windows linux macos rpi aarch64 $MACOS_ARM; do
# Use same values as Go.
if [[ "$source" == "windows" ]]; then
CROSS_OS=windows
CROSS_ARCH=amd64
elif [[ "$source" == "linux" ]]; then
CROSS_OS=linux
CROSS_ARCH=amd64
elif [[ "$source" == "macos" ]]; then
CROSS_OS=darwin
CROSS_ARCH=amd64
elif [[ "$source" == "macos-arm" ]]; then
CROSS_OS=darwin
CROSS_ARCH=aarch64
elif [[ "$source" == "rpi" ]]; then
CROSS_OS=linux
CROSS_ARCH=arm
elif [[ "$source" == "aarch64" ]]; then
CROSS_OS=linux
CROSS_ARCH=arm64
else
echo "Unknown source: $source"
exit 1
fi
mkdir -p vessels/$CROSS_OS/$CROSS_ARCH
cp $source/toit/vessels/vessel* vessels/$CROSS_OS/$CROSS_ARCH
for target in windows linux macos rpi aarch64 $MACOS_ARM; do
if [[ "$source" == "$target" ]]; then
continue
fi
mkdir -p $target/toit/vessels/$CROSS_OS/$CROSS_ARCH
cp $source/toit/vessels/vessel* $target/toit/vessels/$CROSS_OS/$CROSS_ARCH
done
done
- name: Create vessels artifact
run: |
tar c -zf vessels.tar.gz vessels
- name: Download esptools
run: |
for os in windows linux macos rpi aarch64 $MACOS_ARM; do
EXTENSION=""
if [[ "$os" == "windows" ]]; then
EXTENSION=.exe
ESPTOOL_ARCH=windows-x64
elif [[ "$os" == "linux" ]]; then
ESPTOOL_ARCH=linux-x64
elif [[ "$os" == "macos" ]]; then
ESPTOOL_ARCH=macos-x64
elif [[ "$os" == "rpi" ]]; then
ESPTOOL_ARCH=linux-armv7
elif [[ "$os" == "aarch64" ]]; then
ESPTOOL_ARCH=linux-aarch64
elif [[ "$os" == "macos-arm" ]]; then
ESPTOOL_ARCH=macos-aarch64
else
echo "Unknown os: $os"
exit 1
fi
curl -O -L https://github.com/toitlang/esptool/releases/download/$ESPTOOL_VERSION/esptool-$ESPTOOL_ARCH.zip
unzip esptool-$ESPTOOL_ARCH.zip
cp esptool-$ESPTOOL_ARCH/esptool$EXTENSION $os/toit/tools/esptool$EXTENSION
chmod +x $os/toit/tools/esptool$EXTENSION
done
- name: Compress SDKs
run: |
for os in windows linux macos rpi aarch64 $MACOS_ARM; do
tar c -zf toit-$os.tar.gz -C $os toit
done
- name: Add tar-balls with improved naming
run: |
cp toit-linux.tar.gz toit-linux-x64.tar.gz
cp toit-rpi.tar.gz toit-linux-armv7.tar.gz
cp toit-aarch64.tar.gz toit-linux-aarch64.tar.gz
- name: Upload vessels
uses: actions/upload-artifact@v4
with:
name: vessels
path: vessels.tar.gz
- name: Upload Windows unsigned
uses: actions/upload-artifact@v4
with:
name: Windows-build-unsigned-combined
path: toit-windows.tar.gz
- name: Upload macOS unsigned
uses: actions/upload-artifact@v4
with:
name: macOS-build-unsigned-combined
path: ${{ env.MACOS_TARS }}
- name: Upload Linux
uses: actions/upload-artifact@v4
with:
name: Linux-build-combined
path: |
toit-linux-x64.tar.gz
toit-linux-armv7.tar.gz
toit-linux-aarch64.tar.gz
- name: Upload release vessels
if: (env.DO_RELEASE == 'true' && github.repository_owner == 'toitlang')
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: vessels.tar.gz
tag: ${{ env.TOIT_VERSION }}
overwrite: true
- name: Prepare for Linux release upload
if: (env.DO_RELEASE == 'true' && github.repository_owner == 'toitlang')
run: |
mkdir -p linux-upload
cp toit-linux.tar.gz linux-upload
cp toit-rpi.tar.gz linux-upload
cp toit-aarch64.tar.gz linux-upload
cp toit-linux-x64.tar.gz linux-upload
cp toit-linux-armv7.tar.gz linux-upload
cp toit-linux-aarch64.tar.gz linux-upload
- name: Upload Linux release
if: (env.DO_RELEASE == 'true' && github.repository_owner == 'toitlang')
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file_glob: true
file: linux-upload/*
tag: ${{ env.TOIT_VERSION }}
overwrite: true
sign_windows:
runs-on: windows-latest
needs: [combine]
if: (github.event_name == 'release' || startsWith(github.event.inputs.do-release, 'v'))
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with:
name: Windows-build-unsigned-combined
- name: Decompress
run: |
& "C:\Program Files\Git\bin\bash.exe" --noprofile --norc -e -o pipefail -c "tar -xzf toit-windows.tar.gz"
- name: Sign Windows binary
uses: toitlang/action-code-sign@5da128f4fb4f719c1b667867815f6c31e743b111 # v1.1.0
with:
certificate: ${{ secrets.DIGICERT_CERTIFICATE }}
api-key: ${{ secrets.DIGICERT_API_KEY }}
certificate-password: ${{ secrets.DIGICERT_PASSWORD }}
certificate-fingerprint: ${{ secrets.DIGICERT_FINGERPRINT }}
keypair-alias: ${{ vars.DIGICERT_KEYPAIR_ALIAS }}
path: toit/bin
- name: Sign Windows tools
uses: toitlang/action-code-sign@5da128f4fb4f719c1b667867815f6c31e743b111 # v1.1.0
with:
certificate: ${{ secrets.DIGICERT_CERTIFICATE }}
api-key: ${{ secrets.DIGICERT_API_KEY }}
certificate-password: ${{ secrets.DIGICERT_PASSWORD }}
certificate-fingerprint: ${{ secrets.DIGICERT_FINGERPRINT }}
keypair-alias: ${{ vars.DIGICERT_KEYPAIR_ALIAS }}
path: toit/tools
- name: Compress
run: |
& "C:\Program Files\Git\bin\bash.exe" --noprofile --norc -e -o pipefail -c "tar -czf toit-windows.tar.gz toit"
- name: Version number
id: version
shell: powershell
run: |
$versionV = "${{ env.TOIT_VERSION }}"
$version = $versionV.Substring(1)
echo "version=$version" >> $env:GITHUB_OUTPUT
- name: Build installer
run: |
& tools\windows_installer\build.bat ${{ steps.version.outputs.version }} $PWD\toit $PWD\toit-windows-x64-installer.exe
- name: Sign Windows installer
uses: toitlang/action-code-sign@5da128f4fb4f719c1b667867815f6c31e743b111 # v1.1.0
with:
certificate: ${{ secrets.DIGICERT_CERTIFICATE }}
api-key: ${{ secrets.DIGICERT_API_KEY }}
certificate-password: ${{ secrets.DIGICERT_PASSWORD }}
certificate-fingerprint: ${{ secrets.DIGICERT_FINGERPRINT }}
keypair-alias: ${{ vars.DIGICERT_KEYPAIR_ALIAS }}
path: toit-windows-x64-installer.exe
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: Windows-build-signed
path: |
toit-windows-x64.tar.gz
toit-windows-x64-installer.exe
- name: Prepare upload
run: |
mkdir -p windows-upload
cp toit-windows.tar.gz windows-upload/toit-windows.tar.gz
cp toit-windows.tar.gz windows-upload/toit-windows-x64.tar.gz
cp toit-windows-x64-installer.exe windows-upload/toit-windows-x64-installer.exe
- name: Upload release artifacts
if: (env.DO_RELEASE == 'true' && github.repository_owner == 'toitlang')
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file_glob: true
file: windows-upload/*
tag: ${{ env.TOIT_VERSION }}
overwrite: true
sign_macos:
runs-on: macos-latest # An arm64 runner.
needs: [combine]
if: github.event_name == 'release' ||
startsWith(github.event.inputs.do-release, 'v') ||
github.event.inputs.sign-macos == 'true'
steps:
- uses: actions/checkout@v4
- name: Install dependencies
run: |
set -e
brew install create-dmg
brew install zip
- uses: actions/download-artifact@v4
with:
name: macOS-build-unsigned-combined
- name: Create out folder
run: |
mkdir out
- name: Decompress
run: |
tar x -zf toit-macos.tar.gz
mv toit toit-x64
tar x -zf toit-macos-arm.tar.gz
mv toit toit-arm
- name: List executables
id: list-executables
run: |
EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
echo "result<<$EOF" >> $GITHUB_OUTPUT
# Don't sign the esptool which isn't from us.
find toit-x64/bin toit-x64/tools toit-x64/vessels/vessel* toit-x64/vessels/darwin \
toit-arm/bin toit-arm/tools toit-arm/vessels/vessel* toit-arm/vessels/darwin \
-type f \
! -regex '.*/esptool' \
>> $GITHUB_OUTPUT
echo "$EOF" >> $GITHUB_OUTPUT
# Toitware's certificate isn't valid at the moment.
# - name: Sign macOS executables
# uses: toitlang/action-macos-sign-notarize@567fcd7c0b89e1b4d0fbc5132cce6e56224db1b7 # v1.2.0
# with:
# certificate: ${{ secrets.MACOS_CERTIFICATE }}
# certificate-password: ${{ secrets.MACOS_CERTIFICATE_PWD }}
# username: ${{ secrets.AC_USERNAME }}
# password: ${{ secrets.AC_PASSWORD }}
# apple-team-id: ${{ vars.MACOS_TEAM_ID }}
# app-path: ${{ steps.list-executables.outputs.result }}
- name: Compress
run: |
for arch in x64 arm; do
mv toit-$arch toit
tar c -zf out/toit-$arch.tar.gz toit
mv toit toit-$arch
done
- name: Create DMGs
run: |
set -e
ln -s sdk/bin/toit toit-symlink
for arch in x64 arm; do
# Use an empty directory as source so we don't accidentally add other files than SDK.
mkdir empty-$arch
create-dmg \
--volname "jag" \
--add-file toit toit-symlink 0 0 \
--add-file sdk toit-$arch 200 0 \
out/toit-$arch.dmg \
empty-$arch
done
# Toitware's certificate isn't valid at the moment.
# - name: Sign DMGs
# uses: toitlang/action-macos-sign-notarize@567fcd7c0b89e1b4d0fbc5132cce6e56224db1b7 # v1.2.0
# with:
# certificate: ${{ secrets.MACOS_CERTIFICATE }}
# certificate-password: ${{ secrets.MACOS_CERTIFICATE_PWD }}
# username: ${{ secrets.AC_USERNAME }}
# password: ${{ secrets.AC_PASSWORD }}
# apple-team-id: ${{ vars.MACOS_TEAM_ID }}
# app-path: |
# out/toit-x64.dmg
# out/toit-arm.dmg
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: macOS-build-signed
path: |
out/toit-x64.tar.gz
out/toit-arm.tar.gz
out/toit-x64.dmg
out/toit-arm.dmg
- name: Prepare upload
run: |
mkdir -p macos-upload
cp out/toit-x64.tar.gz macos-upload/toit-macos.tar.gz
cp out/toit-x64.tar.gz macos-upload/toit-macos-x64.tar.gz
cp out/toit-arm.tar.gz macos-upload/toit-macos-aarch64.tar.gz
cp out/toit-x64.dmg macos-upload/toit-macos-x64.dmg
cp out/toit-arm.dmg macos-upload/toit-macos-aarch64.dmg
- name: Upload release artifacts x64
if: (env.DO_RELEASE == 'true' && github.repository_owner == 'toitlang')
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file_glob: true
file: macos-upload/*
tag: ${{ env.TOIT_VERSION }}
overwrite: true
external-and-flaky:
needs: build
runs-on: ubuntu-latest
steps:
- name: Set up constants
id: constants
shell: bash
run: |
echo flaky_result_dir=flaky-result >> $GITHUB_OUTPUT
echo flaky_result_name=flaky-result-${{ github.run_id }}-${{ github.run_attempt }} >> $GITHUB_OUTPUT
echo external_result_dir=external-result >> $GITHUB_OUTPUT
echo external_result_name=external-result-${{ github.run_id }}-${{ github.run_attempt }} >> $GITHUB_OUTPUT
- name: Retrieve
uses: actions/download-artifact@v4
with:
pattern: ${{ steps.constants.outputs.flaky_result_name }}*
path: flaky-result
merge-multiple: true
- name: Retrieve
uses: actions/download-artifact@v4
with:
pattern: ${{ steps.constants.outputs.external_result_name }}*
path: external-result
merge-multiple: true
- name: List downloaded files
# This is mostly for debugging purposes.
shell: bash
run: |
find .
- name: Check external
shell: bash
run: |
cat external-result/*
grep "fail" external-result/* && exit 1
# If we get here, we didn't find any failures but
# the exit code of the grep is still 1 or 2. Echo
# something to make it clear and avoid letting the
# result of the action be non-zero.
echo "success: no failures found"
- name: Check flaky
# Even if the external result already failed we want to know whether flaky tests failed
# as well.
# Since the two steps aren't dependent always run this step.
if: always()
shell: bash
run: |
cat flaky-result/*
grep "fail" flaky-result/* && exit 1
# If we get here, we didn't find any failures but
# the exit code of the grep is still 1 or 2. Echo
# something to make it clear and avoid letting the
# result of the action be non-zero.
echo "success: no failures found"
AUR:
name: Update AUR package
runs-on: ubuntu-latest
needs: [combine]
if: (github.event_name == 'release' || startsWith(github.event.inputs.do-release, 'v')) &&
github.repository_owner == 'toitlang'
steps:
# This shouldn't be necessary, but the archlinux-package-action
# doesn't work without it.
- uses: actions/checkout@v4
- name: Ssh
env:
AUR_SSH_KEY: ${{ secrets.AUR_SSH_KEY }}
run: |
echo "$AUR_SSH_KEY" > ${{ github.workspace }}/aur_ssh_key
chmod 600 ${{ github.workspace }}/aur_ssh_key
mkdir -p $HOME/.ssh
echo "Host aur.archlinux.org" >> $HOME/.ssh/config
echo " IdentityFile ${{ github.workspace }}/aur_ssh_key" >> $HOME/.ssh/config
echo " User aur" >> $HOME/.ssh/config
ssh-keyscan -H aur.archlinux.org > $HOME/.ssh/known_hosts
- name: Fetch AUR packages
run: |
git clone ssh://[email protected]/toit.git
git clone ssh://[email protected]/toit-vessels-bin.git
- name: Configure git
run: |
pushd toit
git config user.email "[email protected]"
git config user.name "Leon Gungadin Mogensen"
popd
pushd toit-vessels-bin
git config user.email "[email protected]"
git config user.name "Leon Gungadin Mogensen"
popd
- name: Update AUR packages
shell: bash
run: |
# Toit's PKGBUILD is a bit special in that we don't download
# a tarball (since those don't contain submodules), but instead
# use Git to get the source code.
# For security, we store the hash of the commit in the PKGBUILD,
# which is why we run a custom 'update_hash.sh' script.
# This is the reason we can't just use the 'archlinux-package-action'.
pushd toit
sed -i "s/_tag=.*/_tag=\"${{ env.TOIT_VERSION }}\"/g" PKGBUILD
sed -i "s/pkgrel=.*/pkgrel=1/g" PKGBUILD
./update_hash.sh
popd
# The vessel PKGBUILD also needs the '_tag' replaced instead of
# the usual 'pkgver'.
pushd toit-vessels-bin
sed -i "s/_tag=.*/_tag=\"${{ env.TOIT_VERSION }}\"/g" PKGBUILD
popd
- name: Update .SRCINFO for toit
uses: toitlang/archlinux-package-action@main
with:
path: toit
flags: ''
namcap: false
srcinfo: true
- name: Update pkgsums and .SRCINFO for toit-vessels-bin
uses: toitlang/archlinux-package-action@main
with:
path: toit-vessels-bin
flags: ''
namcap: false
updpkgsums: true
srcinfo: true
- name: Upload toit and toit-vessels-bin
run: |
pushd toit
cat PKGBUILD
cat .SRCINFO
git commit -am "Update to version ${{ env.TOIT_VERSION }}"
git push origin master
popd
pushd toit-vessels-bin
cat PKGBUILD
cat .SRCINFO
git commit -am "Update to version ${{ env.TOIT_VERSION }}"
git push origin master
popd