Add observer script #15
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
## | |
## This workflow handles testing of pull requests and pushes. | |
## It also publishes some packages to any new Erlang/OTP release | |
## | |
## To speed this up it would be nice if one could share docker | |
## images inbetween different jobs, but at the moment this is | |
## not possible so we need to rebuild all of Erlang/OTP multiple | |
## times. | |
## | |
## When ghcr.io support using the GITHUB_TOKEN we should migrate | |
## over to use it instead as that should allow us to use the | |
## built-in caching mechanisms of docker/build-push-action@v2. | |
## However as things are now we use docker directly to make things | |
## work. | |
## | |
name: Build and check Erlang/OTP | |
on: | |
push: | |
pull_request: | |
jobs: | |
pack: | |
name: Build Erlang/OTP (64-bit) | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Create initial pre-release tar | |
run: .github/scripts/init-pre-release.sh | |
- name: Upload source tar archive | |
uses: actions/upload-artifact@v2 | |
with: | |
name: otp_git_archive | |
path: otp_src.tar.gz | |
- name: Docker login | |
uses: docker/login-action@v1 | |
with: | |
registry: docker.pkg.github.com | |
username: ${{ github.repository_owner }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: Calculate BASE image | |
id: base | |
run: | | |
BASE_TAG=$(grep "ARG BASE=" .github/dockerfiles/Dockerfile.64-bit | head -1 | tr '=' ' ' | awk '{print $3}') | |
echo "::set-output name=BASE_TAG::${BASE_TAG}" | |
.github/scripts/base-tag "${BASE_TAG}" | |
- name: Pull BASE image | |
run: docker pull ${{ steps.base.outputs.BASE_TAG }} | |
- name: Build BASE image | |
run: | | |
docker build --pull --tag ${{ steps.base.outputs.BASE_TAG }} \ | |
--cache-from ${{ steps.base.outputs.BASE_TAG }} \ | |
--file .github/dockerfiles/Dockerfile.${{ steps.base.outputs.BASE_TYPE }} \ | |
--build-arg USER=otptest --build-arg GROUP=uucp \ | |
--build-arg uid=$(id -u) \ | |
--build-arg BASE=${{ steps.base.outputs.BASE }} .github/ | |
- name: Build 64-bit image | |
run: | | |
mv otp_src.tar.gz .github/otp.tar.gz | |
docker build --tag otp \ | |
--build-arg MAKEFLAGS=-j$(($(nproc) + 2)) \ | |
--file .github/dockerfiles/Dockerfile.64-bit \ | |
.github/ | |
- name: Save docker image | |
run: docker save otp > otp_docker.tar | |
- name: Upload docker image | |
uses: actions/upload-artifact@v2 | |
with: | |
name: otp_docker | |
path: otp_docker.tar | |
- name: Build pre-built tar archives | |
run: | | |
docker run -v $PWD:/github otp \ | |
"scripts/build-otp-tar -o /github/otp_clean_src.tar.gz /github/otp_src.tar.gz -b /buildroot/otp/ /buildroot/otp.tar.gz" | |
- name: Upload pre-built tar archive | |
uses: actions/upload-artifact@v2 | |
with: | |
name: otp_prebuilt_no_chunks | |
path: otp_src.tar.gz | |
changed-apps: | |
name: Calculate changed applications | |
runs-on: ubuntu-latest | |
outputs: | |
changes: ${{ steps.changes.outputs.changes }} | |
all: ${{ steps.apps.outputs.all }} | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Get applications | |
id: apps | |
run: | | |
.github/scripts/path-filters.sh > .github/scripts/path-filters.yaml | |
ALL_APPS=$(grep '^[a-z_]*:$' .github/scripts/path-filters.yaml | sed 's/:$//') | |
ALL_APPS=$(jq -n --arg inarr "${ALL_APPS}" '$inarr | split("\n")' | tr '\n' ' ') | |
echo "::set-output name=all::${ALL_APPS}" | |
- uses: dorny/paths-filter@v2 | |
id: changes | |
with: | |
filters: .github/scripts/path-filters.yaml | |
build-macos: | |
name: Build Erlang/OTP (macOS) | |
runs-on: macos-10.15 | |
needs: pack | |
env: | |
WXWIDGETS_VERSION: 3.1.5 | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Download source archive | |
uses: actions/download-artifact@v2 | |
with: | |
name: otp_prebuilt_no_chunks | |
- name: Cache wxWidgets | |
id: wxwidgets-cache | |
uses: actions/cache@v2 | |
with: | |
path: wxWidgets | |
key: wxWidgets-${{ env.WXWIDGETS_VERSION }}-${{ runner.os }} | |
- name: Compile wxWidgets | |
if: steps.wxwidgets-cache.outputs.cache-hit != 'true' | |
run: .github/scripts/build-macos-wxwidgets.sh | |
- name: Compile Erlang | |
run: | | |
tar -xzf ./otp_src.tar.gz | |
export PATH=$PWD/wxWidgets/release/bin:$PATH | |
cd otp | |
$GITHUB_WORKSPACE/.github/scripts/build-macos.sh | |
tar -czf otp_macos_$(cat OTP_VERSION)_x86-64.tar.gz -C release . | |
- name: Test Erlang | |
run: | | |
cd otp/release | |
./Install -sasl $PWD | |
./bin/erl -noshell -eval 'io:format("~s", [erlang:system_info(system_version)]), halt().' | |
./bin/erl -noshell -eval 'ok = crypto:start(), io:format("crypto ok~n"), halt().' | |
./bin/erl -noshell -eval '{wx_ref,_,_,_} = wx:new(), io:format("wx ok~n"), halt().' | |
- name: Upload tarball | |
uses: actions/upload-artifact@v2 | |
with: | |
name: otp_prebuilt_macos_x86-64 | |
path: otp/otp_macos_*_x86-64.tar.gz | |
build-ios: | |
env: | |
RELEASE_LIBBEAM: yes | |
TARGET_ARCH: aarch64-apple-ios | |
name: Build Erlang/OTP (iOS) | |
runs-on: macos-10.15 | |
needs: pack | |
steps: | |
- name: Download source archive | |
uses: actions/download-artifact@v2 | |
with: | |
name: otp_prebuilt_no_chunks | |
- name: Compile Erlang | |
run: | | |
tar -xzf ./otp_src.tar.gz | |
cd otp | |
export ERL_TOP=`pwd` | |
export MAKEFLAGS="-j$(($(nproc) + 2)) -O" | |
export ERLC_USE_SERVER=true | |
./otp_build configure --xcomp-conf=./xcomp/erl-xcomp-arm64-ios.conf --without-ssl | |
./otp_build boot -a | |
./otp_build release -a | |
- name: Package .xcframework | |
run: | | |
cd otp | |
export BUILD_ARCH=`./erts/autoconf/config.guess` | |
export LIBS=`find . -not -path "*${BUILD_ARCH}*" -path "*${TARGET_ARCH}*" -not -name "*_st.a" -not -name "*_r.a" -name "*.a" | awk '{ "basename " $1 | getline name; names[name] = $1 } END { for (n in names) { print names[n] }}'` | |
libtool -static -o liberlang.a $LIBS | |
xcodebuild -create-xcframework -output ./liberlang.xcframework -library liberlang.a | |
- name: Upload framework | |
uses: actions/upload-artifact@v2 | |
with: | |
name: ios_framework_${{ env.TARGET_ARCH }} | |
path: otp/liberlang.xcframework | |
build-windows: | |
defaults: | |
run: | |
shell: wsl-bash {0} | |
env: | |
WXWIDGETS_VERSION: 3.1.4 | |
name: Build Erlang/OTP (Windows) | |
runs-on: windows-2022 | |
needs: pack | |
steps: | |
- uses: Vampire/[email protected] | |
with: | |
distribution: Ubuntu-18.04 | |
- name: Install WSL dependencies | |
run: apt update && apt install -y g++-mingw-w64 gcc-mingw-w64 make autoconf unzip | |
- name: Install openssl | |
shell: cmd | |
run: | | |
choco install openssl | |
move "c:\\Program Files\\OpenSSL-Win64" "c:\\OpenSSL-Win64" | |
- name: Cache wxWidgets | |
uses: actions/cache@v2 | |
with: | |
path: wxWidgets | |
key: wxWidgets-${{ env.WXWIDGETS_VERSION }}-${{ runner.os }} | |
# actions/cache on Windows sometimes does not set cache-hit even though there was one. Setting it manually. | |
- name: Set wxWidgets cache | |
id: wxwidgets-cache | |
run: | | |
if [ -d wxWidgets ]; then | |
echo "::set-output name=cache-hit::true" | |
else | |
echo "::set-output name=cache-hit::false" | |
fi | |
- name: Download wxWidgets | |
if: steps.wxwidgets-cache.outputs.cache-hit != 'true' | |
run: | | |
wget https://github.com/wxWidgets/wxWidgets/releases/download/v${{ env.WXWIDGETS_VERSION }}/wxWidgets-${{ env.WXWIDGETS_VERSION }}.zip | |
unzip wxWidgets-${{ env.WXWIDGETS_VERSION }}.zip -d wxWidgets | |
sed -i -r -e 's/wxUSE_POSTSCRIPT +0/wxUSE_POSTSCRIPT 1/' wxWidgets/include/wx/msw/setup.h | |
sed -i -r -e 's/wxUSE_WEBVIEW_EDGE +0/wxUSE_WEBVIEW_EDGE 1/' wxWidgets/include/wx/msw/setup.h | |
- name: Install WebView2 | |
if: steps.wxwidgets-cache.outputs.cache-hit != 'true' | |
shell: cmd | |
run: | | |
cd wxWidgets\\3rdparty | |
nuget install Microsoft.Web.WebView2 -Version 1.0.705.50 -Source https://api.nuget.org/v3/index.json | |
rename Microsoft.Web.WebView2.1.0.705.50 webview2 | |
- name: Build wxWidgets | |
if: steps.wxwidgets-cache.outputs.cache-hit != 'true' | |
shell: cmd | |
run: | | |
cd wxWidgets\\build\\msw | |
call "C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\VC\Auxiliary\\Build\\vcvars64.bat" | |
nmake TARGET_CPU=amd64 BUILD=release SHARED=0 DIR_SUFFIX_CPU= -f makefile.vc | |
- name: Download source archive | |
uses: actions/download-artifact@v2 | |
with: | |
name: otp_prebuilt_no_chunks | |
- name: Compile Erlang | |
run: | | |
mkdir -p /mnt/c/opt/local64/pgm/ | |
cp -R wxWidgets /mnt/c/opt/local64/pgm/wxWidgets-${{ env.WXWIDGETS_VERSION }} | |
tar -xzf ./otp_src.tar.gz | |
cd otp | |
export ERL_TOP=`pwd` | |
export MAKEFLAGS=-j$(($(nproc) + 2)) | |
export ERLC_USE_SERVER=true | |
export ERTS_SKIP_DEPEND=true | |
eval `./otp_build env_win32 x64` | |
./otp_build configure | |
if cat erts/CONF_INFO || cat lib/*/CONF_INFO || cat lib/*/SKIP || cat lib/SKIP-APPLICATIONS; then exit 1; fi | |
./otp_build boot -a | |
./otp_build release -a | |
cp /mnt/c/opt/local64/pgm/wxWidgets-${{ env.WXWIDGETS_VERSION }}/3rdparty/webview2/runtimes/win-x64/native/WebView2Loader.dll $ERL_TOP/release/win32/erts-*/bin/ | |
./otp_build installer_win32 | |
- name: Upload installer | |
uses: actions/upload-artifact@v2 | |
with: | |
name: otp_win32_installer | |
path: otp/release/win32/otp*.exe | |
build: | |
name: Build Erlang/OTP | |
runs-on: ubuntu-latest | |
needs: pack | |
strategy: | |
matrix: | |
type: [32-bit,cross-compile,clang] | |
fail-fast: false | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Download source archive | |
uses: actions/download-artifact@v2 | |
with: | |
name: otp_prebuilt_no_chunks | |
- run: echo "${{ needs.pack.outputs.changes }}" | |
- name: Docker login | |
uses: docker/login-action@v1 | |
with: | |
registry: docker.pkg.github.com | |
username: ${{ github.repository_owner }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: Calculate BASE image | |
id: base | |
run: | | |
BASE_TAG=$(grep "ARG BASE=" .github/dockerfiles/Dockerfile.${{ matrix.type }} | head -1 | tr '=' ' ' | awk '{print $3}') | |
echo "::set-output name=BASE_TAG::${BASE_TAG}" | |
.github/scripts/base-tag "${BASE_TAG}" | |
- name: Pull BASE image | |
run: docker pull ${{ steps.base.outputs.BASE_TAG }} | |
- name: Build BASE image | |
run: | | |
docker build --pull --tag ${{ steps.base.outputs.BASE_TAG }} \ | |
--cache-from ${{ steps.base.outputs.BASE_TAG }} \ | |
--file .github/dockerfiles/Dockerfile.${{ steps.base.outputs.BASE_TYPE }} \ | |
--build-arg USER=otptest --build-arg GROUP=uucp \ | |
--build-arg uid=$(id -u) \ | |
--build-arg BASE=${{ steps.base.outputs.BASE }} .github/ | |
- name: Build ${{ matrix.type }} image | |
run: | | |
mv otp_src.tar.gz .github/otp.tar.gz | |
docker build --tag otp --file .github/dockerfiles/Dockerfile.${{ matrix.type }} \ | |
.github/ | |
documentation: | |
name: Build and check documentation | |
runs-on: ubuntu-latest | |
needs: pack | |
steps: | |
## Download docker images | |
- name: Download source archive | |
uses: actions/download-artifact@v2 | |
with: | |
name: otp_docker | |
- name: Restore docker image | |
run: docker load < otp_docker.tar | |
## Build pre-built tar with chunks | |
- name: Build doc chunks | |
run: | | |
docker build -t otp - <<EOF | |
FROM otp | |
RUN make docs DOC_TARGETS=chunks | |
EOF | |
- name: Build pre-built tar archives | |
run: | | |
docker run -v $PWD:/github otp \ | |
"scripts/build-otp-tar -o /github/otp_clean_src.tar.gz /github/otp_src.tar.gz -b /buildroot/otp/ /buildroot/otp.tar.gz" | |
- name: Upload pre-built tar archive | |
uses: actions/upload-artifact@v2 | |
with: | |
name: otp_prebuilt | |
path: otp_src.tar.gz | |
## Build all the documentation | |
- name: Build documentation | |
run: | | |
docker build -t otp - <<EOF | |
FROM otp | |
RUN make docs | |
EOF | |
- name: Release docs to publish | |
run: | | |
docker run -v $PWD/:/github otp \ | |
"make release docs release_docs && make release_docs DOC_TARGETS='man html pdf' RELEASE_ROOT=/github/docs" | |
sudo chown -R `whoami` docs | |
cd docs | |
tar czf ../otp_doc_man.tar.gz man | |
rm -rf man | |
tar czf ../otp_doc_html.tar.gz * | |
- name: Upload html documentation archive | |
uses: actions/upload-artifact@v2 | |
with: | |
name: otp_doc_html | |
path: otp_doc_html.tar.gz | |
- name: Upload man documentation archive | |
uses: actions/upload-artifact@v2 | |
with: | |
name: otp_doc_man | |
path: otp_doc_man.tar.gz | |
## Documentation checks | |
- name: Run xmllint | |
run: docker run otp "make xmllint" | |
- name: Run html link check | |
run: docker run -v $PWD/:/github otp "scripts/otp_html_check /github/docs doc/index.html" | |
static: | |
name: Run static analysis | |
runs-on: ubuntu-latest | |
needs: pack | |
steps: | |
## Download docker images | |
- name: Download source archive | |
uses: actions/download-artifact@v2 | |
with: | |
name: otp_docker | |
- name: Restore docker image | |
run: docker load < otp_docker.tar | |
- name: Install clang-format | |
run: | | |
docker build -t otp - <<EOF | |
FROM otp | |
RUN sudo apt-get install -y clang-format | |
EOF | |
## Check formatting of cpp code | |
- name: Check format | |
run: docker run otp "make format-check" | |
## Run dialyzer | |
- name: Run dialyzer | |
run: docker run otp 'scripts/run-dialyzer' | |
test: | |
name: Test Erlang/OTP | |
runs-on: ubuntu-latest | |
needs: [pack, changed-apps] | |
if: needs.changed-apps.outputs.changes != '[]' | |
strategy: | |
matrix: | |
# type: ${{ fromJson(needs.changed-apps.outputs.all) }} | |
type: ${{ fromJson(needs.changed-apps.outputs.changes) }} | |
# type: ["os_mon","sasl"] | |
fail-fast: false | |
steps: | |
## Download docker images | |
- name: Download source archive | |
uses: actions/download-artifact@v2 | |
with: | |
name: otp_docker | |
- name: Restore docker image | |
run: docker load < otp_docker.tar | |
- name: Run tests | |
id: run-tests | |
run: | | |
set -x | |
mkdir $PWD/make_test_dir | |
APP="${{ matrix.type }}" | |
## Need to specialize for epmd, emulator and debug | |
case "${APP}" in | |
emulator) DIR=erts/emulator/ ;; | |
epmd) DIR=erts/epmd ;; | |
debug) DIR=lib/os_mon; APP=os_mon; TYPE=debug ;; | |
*) DIR=lib/${{ matrix.type }} ;; | |
esac | |
sudo service apport stop | |
sudo bash -c "echo 'core.%p' > /proc/sys/kernel/core_pattern" | |
docker run --ulimit core=-1 --ulimit nofile=5000:5000 --pids-limit 512 \ | |
-e CTRUN_TIMEOUT=90 -e SPEC_POSTFIX=gh \ | |
-e EXTRA_ARGS="-ct_hooks cth_surefire [{path,\"/buildroot/otp/$DIR/make_test_dir/${{ matrix.type }}_junit.xml\"}]" \ | |
-v "$PWD/make_test_dir:/buildroot/otp/$DIR/make_test_dir" \ | |
otp "make TYPE=${TYPE} && make ${APP}_test TYPE=${TYPE}" | |
## Rename os_mon to debug for debug build | |
if [ "$APP" != "${{ matrix.type }}" ]; then | |
mv make_test_dir/${APP}_test "make_test_dir/${{ matrix.type }}_test" | |
fi | |
- name: Cleanup tests | |
if: always() | |
run: | | |
rm -rf make_test_dir/otp || true | |
sudo bash -c "chown -R `whoami` make_test_dir && chmod -R +r make_test_dir" | |
tar czf ${{ matrix.type }}_test_results.tar.gz make_test_dir | |
- name: Upload test results | |
uses: actions/upload-artifact@v2 | |
if: always() | |
with: | |
name: ${{ matrix.type }}_test_results | |
path: ${{ matrix.type }}_test_results.tar.gz | |
system-test: | |
name: Test Erlang/OTP (system) | |
runs-on: ubuntu-latest | |
if: always() # Run even if the need has failed | |
needs: test | |
steps: | |
- name: Download test results | |
uses: actions/download-artifact@v2 | |
- name: Merge test results | |
run: | | |
shopt -s nullglob | |
mkdir -p make_test_dir | |
for file in *_test_results/*.tar.gz; do | |
tar xzf $file | |
done | |
docker load < otp_docker/otp_docker.tar | |
docker run -v $PWD/make_test_dir:/buildroot/otp/erts/make_test_dir otp \ | |
"ct_run -refresh_logs /buildroot/otp/erts/make_test_dir" | |
- name: Run system tests | |
run: | | |
docker run --ulimit core=-1 --ulimit nofile=5000:5000 --pids-limit 512 \ | |
-e CTRUN_TIMEOUT=90 -e SPEC_POSTFIX=gh \ | |
-e EXTRA_ARGS="-ct_hooks cth_surefire [{path,\"/buildroot/otp/erts/make_test_dir/system_junit.xml\"}]" \ | |
-e OTP_DAILY_BUILD_TOP_DIR=/buildroot/otp/erts/make_test_dir \ | |
-v $PWD/make_test_dir:/buildroot/otp/erts/make_test_dir otp \ | |
"make system_test" | |
- name: Cleanup tests | |
if: always() | |
run: | | |
rm -rf make_test_dir/otp || true | |
tar czf test_results.tar.gz make_test_dir | |
# Translate file="/buildroot/otp/lib/os_mon/make_test_dir/os_mon_test/cpu_sup_SUITE.erl" | |
# to file="lib/os_mon/test/cpu_sup_SUITE.erl" | |
sed -i -e 's:file="/buildroot/otp/:file=":g' \ | |
-e 's:\(file="[^/]*/[^/]*/\)make_test_dir/[^/]*:\1test:g' \ | |
-e 's:\(file="erts/\)make_test_dir/[^/]*:\1test:g' \ | |
make_test_dir/*_junit.xml | |
- name: Upload test results | |
uses: actions/upload-artifact@v2 | |
if: always() | |
with: | |
name: test_results | |
path: test_results.tar.gz | |
- name: Upload Test Results | |
if: always() | |
uses: actions/upload-artifact@v2 | |
with: | |
name: Unit Test Results | |
path: | | |
make_test_dir/*_junit.xml | |
## If this is a tag that has been pushed we do some release work | |
release: | |
name: Release Erlang/OTP | |
runs-on: ubuntu-latest | |
needs: documentation | |
if: startsWith(github.ref, 'refs/tags/') && github.repository == 'erlang/otp' | |
steps: | |
## This step outputs the tag name and whether the tag is a release or patch | |
## (all releases have only two version identifiers, while patches have three | |
## or more) | |
- name: Get Tag Name | |
id: tag | |
run: | | |
TAG=${GITHUB_REF#refs/tags/} | |
VSN=${TAG#OTP-} | |
echo "::set-output name=tag::${TAG}" | |
echo "::set-output name=vsn::${VSN}" | |
- uses: actions/checkout@v2 | |
## Publish the pre-built archive and docs | |
- name: Download source archive | |
uses: actions/download-artifact@v2 | |
with: | |
name: otp_prebuilt | |
- name: Download html docs | |
uses: actions/download-artifact@v2 | |
with: | |
name: otp_doc_html | |
- name: Download man docs | |
uses: actions/download-artifact@v2 | |
with: | |
name: otp_doc_man | |
## We add the correct version name into the file names | |
## and create the hash files for all assets | |
- name: Create pre-build and doc archives | |
run: .github/scripts/create-artifacts.sh artifacts ${{ steps.tag.outputs.tag }} | |
## Create hash files | |
- name: Create pre-build and doc archives | |
run: | | |
shopt -s nullglob | |
cd artifacts | |
FILES=$(ls {*.tar.gz,*.txt}) | |
md5sum $FILES > MD5.txt | |
sha256sum $FILES > SHA256.txt | |
- name: Upload pre-built and doc tar archives | |
uses: softprops/action-gh-release@v1 | |
with: | |
name: OTP ${{ steps.tag.outputs.vsn }} | |
files: | | |
artifacts/*.tar.gz | |
artifacts/*.txt | |
env: | |
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
- name: Deploy on erlang.org | |
env: | |
GITHUB_TOKEN: ${{ secrets.TRIGGER_ERLANG_ORG_BUILD }} | |
run: | | |
curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/erlang/erlang-org/actions/workflows/update-gh-cache.yaml/dispatches" -d '{"ref":"master"}' | |
event_file: | |
name: "Event File" | |
runs-on: ubuntu-latest | |
steps: | |
- name: Upload | |
uses: actions/upload-artifact@v2 | |
with: | |
name: Event File | |
path: ${{ github.event_path }} |