From 45d0c6a2476ba42ef5da3a08e7a40df868a3b04b Mon Sep 17 00:00:00 2001 From: Kohei Tokunaga Date: Wed, 13 Mar 2024 12:40:39 +0900 Subject: [PATCH] tmp Signed-off-by: Kohei Tokunaga --- .github/workflows/tests.yml | 31 ++++++++++----------- tests/benchmark-pull/run.sh | 52 +++++++++++++++++++++--------------- tests/benchmark-pull/targets | 6 +++++ 3 files changed, 50 insertions(+), 39 deletions(-) create mode 100644 tests/benchmark-pull/targets diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 17a3b27b..31e4dcdb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,23 +47,19 @@ jobs: benchmark-pull: runs-on: ubuntu-22.04 name: Benchmark - strategy: - fail-fast: false - matrix: - target: - - { name: "gcc-9.2.0-esgz", image: "ghcr.io/stargz-containers/gcc:9.2.0-esgz", want: "hello", command: "[\"echo\", \"hello\"]" } - - { name: "gcc-9.2.0-org", image: "ghcr.io/stargz-containers/gcc:9.2.0-org", want: "hello", command: "[\"echo\", \"hello\"]" } - - { name: "python-3.10-esgz", image: "ghcr.io/stargz-containers/python:3.10-esgz", want: "hello", command: "[\"/bin/sh\", \"-c\", \"python -c 'print(\\\"hello2\\\")'\"]" } - - { name: "python-3.10-org", image: "ghcr.io/stargz-containers/python:3.10-org", want: "hello", command: "[\"/bin/sh\", \"-c\", \"python -c 'print(\\\"hello2\\\")'\"]" } env: - TARGET_IMAGE: ${{ matrix.target.image }} - TARGET_STRING: ${{ matrix.target.want }} - TARGET_COMMAND: ${{ matrix.target.command }} PERCENTILE: 90 SAMPLES_NUM: 10 BENCHMARK_RESULT_DIR: ${{ github.workspace }}/benchmark/ - BENCHMARK_RESULT: ${{ github.workspace }}/benchmark/result steps: + - name: Prepare dir + run: | + mkdir -p "${{ env.BENCHMARK_RESULT_DIR }}" + - name: Get instance information + run: | + curl -H "Metadata:true" "http://169.254.169.254/metadata/instance?api-version=2019-11-01" | \ + jq '{ location : .compute.location, vmSize : .compute.vmSize }' | \ + tee ${{ env.BENCHMARK_RESULT_DIR }}/instance.json - name: Install tools run: | pip install numpy @@ -71,15 +67,16 @@ jobs: with: go-version: '1.21.x' - uses: actions/checkout@v4 - - name: Benchmark + - name: run working-directory: tests/benchmark-pull - run: | - mkdir -p "${{ env.BENCHMARK_RESULT_DIR }}" - ./run.sh + env: + BENCHMARK_RESULT: ${{ env.BENCHMARK_RESULT_DIR }}/result + TARGET_FILE: targets + run: ./run.sh - uses: actions/upload-artifact@v3 if: ${{ always() }} with: - name: "benchmarking-result-${{ matrix.target.name }}" + name: "benchmarking-result" path: ${{ env.BENCHMARK_RESULT_DIR }} # benchmark: diff --git a/tests/benchmark-pull/run.sh b/tests/benchmark-pull/run.sh index 74663084..382259ad 100755 --- a/tests/benchmark-pull/run.sh +++ b/tests/benchmark-pull/run.sh @@ -5,6 +5,7 @@ set -euo pipefail SAMPLES_NUM_="${SAMPLES_NUM:-3}" PERCENTILE_="${PERCENTILE:-90}" BENCHMARK_RESULT_="${BENCHMARK_RESULT:-}" +TARGET_FILE_="${TARGET_FILE:-}" CALCTEMP=$(mktemp) function percentile { @@ -33,8 +34,6 @@ print(np.percentile(a=np.array(arr), q=${PERCENTILE_}, interpolation='linear')) EOF } -echo "Using image at ${TARGET_IMAGE} (${PERCENTILE} percentile, ${SAMPLES_NUM_} samples) (want=${TARGET_STRING}, command=${TARGET_COMMAND})" - pushd ./../../ make c2w imagemounter.wasm ./out/c2w --external-bundle ./out/out.wasm @@ -62,26 +61,35 @@ function cleanup { } trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM -for I in $(seq 1 ${SAMPLES_NUM_}); do - docker compose up -d --force-recreate - sleep 5 - docker exec benchmark-pull-benchmark-1 python /run.py "image=${TARGET_IMAGE}&want=${TARGET_STRING}&command=$(echo -n ${TARGET_COMMAND} | base64)" | tee "${TMP_RES}" - START_TIME=$(cat "${TMP_RES}" | grep "start timer" | yq -j eval . - | jq -r ".message" | sed -E 's/[^"]*(.*)/\1/' | sed -E 's/\\\\/\\/g' | jq -r | jq -r ".time") - PULL_TIME=$(cat "${TMP_RES}" | grep "detected pull completion" | yq -j eval . - | jq -r ".message" | sed -E 's/[^"]*(.*)/{\1/' | sed -E 's/\\\\/\\/g' | jq -r | jq -r ".time") - RUN_TIME=$(cat "${TMP_RES}" | grep "found wanted string" | yq -j eval . - | jq -r ".message" | sed -E 's/[^"]*(.*)/\1/' | sed -E 's/\\\\/\\/g' | jq -r | jq -r ".time") +ELEMS=$(jq length "${TARGET_FILE_}") + +for ((e = 0 ; e < ${ELEMS} ; e++ )); do + TARGET_IMAGE=$(jq -r '.['"$e"'].image' "${TARGET_FILE_}") + TARGET_STRING=$(jq -r '.['"$e"'].want' "${TARGET_FILE_}") + TARGET_COMMAND=$(jq -r '.['"$e"'].command' "${TARGET_FILE_}") + echo "Using image at ${TARGET_IMAGE} (${PERCENTILE} percentile, ${SAMPLES_NUM_} samples) (want=${TARGET_STRING}, command=${TARGET_COMMAND})" + for I in $(seq 1 ${SAMPLES_NUM_}); do + docker compose up -d --force-recreate + sleep 5 + docker exec benchmark-pull-benchmark-1 python /run.py "image=${TARGET_IMAGE}&want=${TARGET_STRING}&command=$(echo -n ${TARGET_COMMAND} | base64)" | tee "${TMP_RES}" + START_TIME=$(cat "${TMP_RES}" | grep "start timer" | yq -j eval . - | jq -r ".message" | sed -E 's/[^"]*(.*)/\1/' | sed -E 's/\\\\/\\/g' | jq -r | jq -r ".time") + PULL_TIME=$(cat "${TMP_RES}" | grep "detected pull completion" | yq -j eval . - | jq -r ".message" | sed -E 's/[^"]*(.*)/{\1/' | sed -E 's/\\\\/\\/g' | jq -r | jq -r ".time") + RUN_TIME=$(cat "${TMP_RES}" | grep "found wanted string" | yq -j eval . - | jq -r ".message" | sed -E 's/[^"]*(.*)/\1/' | sed -E 's/\\\\/\\/g' | jq -r | jq -r ".time") - PULL_DURATION=$(expr $(date -d"${PULL_TIME}" +%s) - $(date -d"${START_TIME}" +%s)) - RUN_DURATION=$(expr $(date -d"${RUN_TIME}" +%s) - $(date -d"${START_TIME}" +%s)) + PULL_DURATION=$(expr $(date -d"${PULL_TIME}" +%s) - $(date -d"${START_TIME}" +%s)) + RUN_DURATION=$(expr $(date -d"${RUN_TIME}" +%s) - $(date -d"${START_TIME}" +%s)) - echo "{\"pull\": \""${PULL_DURATION}"\", \"total\": \""${RUN_DURATION}"\"}" | tee -a "${TMP_DATA_LOG}" - echo "${PULL_DURATION}" | tee -a "${TMP_DATA_PULL}" - echo "${RUN_DURATION}" | tee -a "${TMP_DATA_RUN}" + echo "{\"pull\": \""${PULL_DURATION}"\", \"total\": \""${RUN_DURATION}"\"}" | tee "${TMP_DATA_LOG}" + echo "${PULL_DURATION}" | tee -a "${TMP_DATA_PULL}" + echo "${RUN_DURATION}" | tee -a "${TMP_DATA_RUN}" + done + echo "======== RESULT (${PERCENTILE} percentile, ${SAMPLES_NUM_} samples) =========" + RES_PULL=$(percentile "${TMP_DATA_PULL}" "${SAMPLES_NUM_}") + RES_RUN=$(percentile "${TMP_DATA_RUN}" "${SAMPLES_NUM_}") + echo "pull: ${RES_PULL}, total: ${RES_RUN}" + if [ "${BENCHMARK_RESULT_}" != "" ] ; then + echo "======== RESULT ${TARGET_IMAGE} (want=${TARGET_STRING}, command=${TARGET_COMMAND}) (${PERCENTILE} percentile, ${SAMPLES_NUM_} samples) =========" >> "${BENCHMARK_RESULT_}" + cat "${TMP_DATA_LOG}" >> "${BENCHMARK_RESULT_}" + echo "RESULT: pull: ${RES_PULL}, total: ${RES_RUN}" >> "${BENCHMARK_RESULT_}" + fi done -echo "======== RESULT (${PERCENTILE} percentile, ${SAMPLES_NUM_} samples) =========" -RES_PULL=$(percentile "${TMP_DATA_PULL}" "${SAMPLES_NUM_}") -RES_RUN=$(percentile "${TMP_DATA_RUN}" "${SAMPLES_NUM_}") -echo "pull: ${RES_PULL}, total: ${RES_RUN}" -if [ "${BENCHMARK_RESULT_}" != "" ] ; then - cat "${TMP_DATA_LOG}" > "${BENCHMARK_RESULT_}" - echo "RESULT: pull: ${RES_PULL}, total: ${RES_RUN}" >> "${BENCHMARK_RESULT_}" -fi diff --git a/tests/benchmark-pull/targets b/tests/benchmark-pull/targets new file mode 100644 index 00000000..d5130fbb --- /dev/null +++ b/tests/benchmark-pull/targets @@ -0,0 +1,6 @@ +[ + { "name": "gcc-9.2.0-esgz", "image": "ghcr.io/stargz-containers/gcc:9.2.0-esgz", "want": "hello", "command": "[\"echo\", \"hello\"]" }, + { "name": "gcc-9.2.0-org", "image": "ghcr.io/stargz-containers/gcc:9.2.0-org", "want": "hello", "command": "[\"echo\", \"hello\"]" }, + { "name": "python-3.10-esgz", "image": "ghcr.io/stargz-containers/python:3.10-esgz", "want": "hello", "command": "[\"/bin/sh\", \"-c\", \"python -c 'print(\\\"hello2\\\")'\"]" }, + { "name": "python-3.10-org", "image": "ghcr.io/stargz-containers/python:3.10-org", "want": "hello", "command": "[\"/bin/sh\", \"-c\", \"python -c 'print(\\\"hello2\\\")'\"]" } +]