tmp

Signed-off-by: Kohei Tokunaga <[email protected]>
ktock · Mar 13, 2024 · 45d0c6a · 45d0c6a
1 parent e7dd861
commit 45d0c6a
Show file tree

Hide file tree

Showing 3 changed files with 50 additions and 39 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -47,39 +47,36 @@ jobs:
   benchmark-pull:
     runs-on: ubuntu-22.04
     name: Benchmark
-    strategy:
-      fail-fast: false
-      matrix:
-        target:
-        - { name: "gcc-9.2.0-esgz", image: "ghcr.io/stargz-containers/gcc:9.2.0-esgz", want: "hello", command: "[\"echo\", \"hello\"]" }
-        - { name: "gcc-9.2.0-org", image: "ghcr.io/stargz-containers/gcc:9.2.0-org", want: "hello", command: "[\"echo\", \"hello\"]" }
-        - { name: "python-3.10-esgz", image: "ghcr.io/stargz-containers/python:3.10-esgz", want: "hello", command: "[\"/bin/sh\", \"-c\", \"python -c 'print(\\\"hello2\\\")'\"]" }
-        - { name: "python-3.10-org", image: "ghcr.io/stargz-containers/python:3.10-org", want: "hello", command: "[\"/bin/sh\", \"-c\", \"python -c 'print(\\\"hello2\\\")'\"]" }
     env:
-      TARGET_IMAGE: ${{ matrix.target.image }}
-      TARGET_STRING: ${{ matrix.target.want }}
-      TARGET_COMMAND: ${{ matrix.target.command }}
       PERCENTILE: 90
       SAMPLES_NUM: 10
       BENCHMARK_RESULT_DIR: ${{ github.workspace }}/benchmark/
-      BENCHMARK_RESULT: ${{ github.workspace }}/benchmark/result
     steps:
+    - name: Prepare dir
+      run: |
+        mkdir -p "${{ env.BENCHMARK_RESULT_DIR }}"
+    - name: Get instance information
+      run: |
+          curl -H "Metadata:true" "http://169.254.169.254/metadata/instance?api-version=2019-11-01" | \
+          jq '{ location : .compute.location, vmSize : .compute.vmSize }' | \
+          tee ${{ env.BENCHMARK_RESULT_DIR }}/instance.json
     - name: Install tools
       run: |
           pip install numpy
     - uses: actions/setup-go@v5
       with:
         go-version: '1.21.x'
     - uses: actions/checkout@v4
-    - name: Benchmark
+    - name: run
       working-directory: tests/benchmark-pull
-      run: |
-        mkdir -p "${{ env.BENCHMARK_RESULT_DIR }}"
-        ./run.sh
+      env:
+        BENCHMARK_RESULT: ${{ env.BENCHMARK_RESULT_DIR }}/result
+        TARGET_FILE: targets
+      run: ./run.sh
     - uses: actions/upload-artifact@v3
       if: ${{ always() }}
       with:
-        name: "benchmarking-result-${{ matrix.target.name }}"
+        name: "benchmarking-result"
         path: ${{ env.BENCHMARK_RESULT_DIR }}
 
   # benchmark:

diff --git a/tests/benchmark-pull/run.sh b/tests/benchmark-pull/run.sh
@@ -5,6 +5,7 @@ set -euo pipefail
 SAMPLES_NUM_="${SAMPLES_NUM:-3}"
 PERCENTILE_="${PERCENTILE:-90}"
 BENCHMARK_RESULT_="${BENCHMARK_RESULT:-}"
+TARGET_FILE_="${TARGET_FILE:-}"
 
 CALCTEMP=$(mktemp)
 function percentile {
@@ -33,8 +34,6 @@ print(np.percentile(a=np.array(arr), q=${PERCENTILE_}, interpolation='linear'))
 EOF
 }
 
-echo "Using image at ${TARGET_IMAGE} (${PERCENTILE} percentile, ${SAMPLES_NUM_} samples) (want=${TARGET_STRING}, command=${TARGET_COMMAND})"
-
 pushd ./../../
 make c2w imagemounter.wasm
 ./out/c2w --external-bundle ./out/out.wasm
@@ -62,26 +61,35 @@ function cleanup {
 }
 trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM
 
-for I in $(seq 1 ${SAMPLES_NUM_}); do 
-    docker compose up -d --force-recreate
-    sleep 5
-    docker exec benchmark-pull-benchmark-1 python /run.py "image=${TARGET_IMAGE}&want=${TARGET_STRING}&command=$(echo -n ${TARGET_COMMAND} | base64)" | tee "${TMP_RES}"
-    START_TIME=$(cat "${TMP_RES}" | grep "start timer" | yq -j eval . - | jq -r ".message" | sed -E 's/[^"]*(.*)/\1/' | sed -E 's/\\\\/\\/g'  | jq -r | jq -r ".time")
-    PULL_TIME=$(cat "${TMP_RES}" | grep "detected pull completion" | yq -j eval . - | jq -r ".message" | sed -E 's/[^"]*(.*)/{\1/' | sed -E 's/\\\\/\\/g'  | jq -r | jq -r ".time")
-    RUN_TIME=$(cat "${TMP_RES}" | grep "found wanted string" | yq -j eval . - | jq -r ".message" | sed -E 's/[^"]*(.*)/\1/' | sed -E 's/\\\\/\\/g'  | jq -r | jq -r ".time")
+ELEMS=$(jq length "${TARGET_FILE_}")
+
+for ((e = 0 ; e < ${ELEMS} ; e++ )); do
+    TARGET_IMAGE=$(jq -r '.['"$e"'].image' "${TARGET_FILE_}")
+    TARGET_STRING=$(jq -r '.['"$e"'].want' "${TARGET_FILE_}")
+    TARGET_COMMAND=$(jq -r '.['"$e"'].command' "${TARGET_FILE_}")
+    echo "Using image at ${TARGET_IMAGE} (${PERCENTILE} percentile, ${SAMPLES_NUM_} samples) (want=${TARGET_STRING}, command=${TARGET_COMMAND})"
+    for I in $(seq 1 ${SAMPLES_NUM_}); do 
+        docker compose up -d --force-recreate
+        sleep 5
+        docker exec benchmark-pull-benchmark-1 python /run.py "image=${TARGET_IMAGE}&want=${TARGET_STRING}&command=$(echo -n ${TARGET_COMMAND} | base64)" | tee "${TMP_RES}"
+        START_TIME=$(cat "${TMP_RES}" | grep "start timer" | yq -j eval . - | jq -r ".message" | sed -E 's/[^"]*(.*)/\1/' | sed -E 's/\\\\/\\/g'  | jq -r | jq -r ".time")
+        PULL_TIME=$(cat "${TMP_RES}" | grep "detected pull completion" | yq -j eval . - | jq -r ".message" | sed -E 's/[^"]*(.*)/{\1/' | sed -E 's/\\\\/\\/g'  | jq -r | jq -r ".time")
+        RUN_TIME=$(cat "${TMP_RES}" | grep "found wanted string" | yq -j eval . - | jq -r ".message" | sed -E 's/[^"]*(.*)/\1/' | sed -E 's/\\\\/\\/g'  | jq -r | jq -r ".time")
 
-    PULL_DURATION=$(expr $(date -d"${PULL_TIME}" +%s) - $(date -d"${START_TIME}" +%s))
-    RUN_DURATION=$(expr $(date -d"${RUN_TIME}" +%s) - $(date -d"${START_TIME}" +%s))
+        PULL_DURATION=$(expr $(date -d"${PULL_TIME}" +%s) - $(date -d"${START_TIME}" +%s))
+        RUN_DURATION=$(expr $(date -d"${RUN_TIME}" +%s) - $(date -d"${START_TIME}" +%s))
 
-    echo "{\"pull\": \""${PULL_DURATION}"\", \"total\": \""${RUN_DURATION}"\"}" | tee -a "${TMP_DATA_LOG}"
-    echo "${PULL_DURATION}" | tee -a "${TMP_DATA_PULL}"
-    echo "${RUN_DURATION}" | tee -a "${TMP_DATA_RUN}"
+        echo "{\"pull\": \""${PULL_DURATION}"\", \"total\": \""${RUN_DURATION}"\"}" | tee "${TMP_DATA_LOG}"
+        echo "${PULL_DURATION}" | tee -a "${TMP_DATA_PULL}"
+        echo "${RUN_DURATION}" | tee -a "${TMP_DATA_RUN}"
+    done
+    echo "======== RESULT (${PERCENTILE} percentile, ${SAMPLES_NUM_} samples) ========="
+    RES_PULL=$(percentile "${TMP_DATA_PULL}" "${SAMPLES_NUM_}")
+    RES_RUN=$(percentile "${TMP_DATA_RUN}" "${SAMPLES_NUM_}")
+    echo "pull: ${RES_PULL}, total: ${RES_RUN}"
+    if [ "${BENCHMARK_RESULT_}" != "" ] ; then
+        echo "======== RESULT ${TARGET_IMAGE} (want=${TARGET_STRING}, command=${TARGET_COMMAND}) (${PERCENTILE} percentile, ${SAMPLES_NUM_} samples) =========" >> "${BENCHMARK_RESULT_}"
+        cat "${TMP_DATA_LOG}" >> "${BENCHMARK_RESULT_}"
+        echo "RESULT: pull: ${RES_PULL}, total: ${RES_RUN}" >> "${BENCHMARK_RESULT_}"
+    fi
 done
-echo "======== RESULT (${PERCENTILE} percentile, ${SAMPLES_NUM_} samples) ========="
-RES_PULL=$(percentile "${TMP_DATA_PULL}" "${SAMPLES_NUM_}")
-RES_RUN=$(percentile "${TMP_DATA_RUN}" "${SAMPLES_NUM_}")
-echo "pull: ${RES_PULL}, total: ${RES_RUN}"
-if [ "${BENCHMARK_RESULT_}" != "" ] ; then
-    cat "${TMP_DATA_LOG}" > "${BENCHMARK_RESULT_}"
-    echo "RESULT: pull: ${RES_PULL}, total: ${RES_RUN}" >> "${BENCHMARK_RESULT_}"
-fi
diff --git a/tests/benchmark-pull/targets b/tests/benchmark-pull/targets
@@ -0,0 +1,6 @@
+[
+    { "name": "gcc-9.2.0-esgz", "image": "ghcr.io/stargz-containers/gcc:9.2.0-esgz", "want": "hello", "command": "[\"echo\", \"hello\"]" },
+    { "name": "gcc-9.2.0-org", "image": "ghcr.io/stargz-containers/gcc:9.2.0-org", "want": "hello", "command": "[\"echo\", \"hello\"]" },
+    { "name": "python-3.10-esgz", "image": "ghcr.io/stargz-containers/python:3.10-esgz", "want": "hello", "command": "[\"/bin/sh\", \"-c\", \"python -c 'print(\\\"hello2\\\")'\"]" },
+    { "name": "python-3.10-org", "image": "ghcr.io/stargz-containers/python:3.10-org", "want": "hello", "command": "[\"/bin/sh\", \"-c\", \"python -c 'print(\\\"hello2\\\")'\"]" }
+]