Mobile Comparative Benchmarks #681
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2023 The OpenXLA Authors | |
| # | |
| # Licensed under the Apache License v2.0 with LLVM Exceptions. | |
| # See https://llvm.org/LICENSE.txt for license information. | |
| # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| # | |
| # Mobile Comparative Benchmarks Workflow. | |
| name: Mobile Comparative Benchmarks | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| # Scheduled to run at 09:00 UTC and 21:00 UTC. | |
| - cron: '0 09,21 * * *' | |
| concurrency: | |
| # A PR number if a pull request and otherwise the commit hash. This cancels | |
| # queued and in-progress runs for the same PR (presubmit) or commit | |
| # (postsubmit). | |
| group: ${{ github.workflow }}-${{ github.event.number || github.sha }} | |
| cancel-in-progress: true | |
| env: | |
| GCS_DIR: gs://openxla-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }} | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| runner-group: ${{ steps.configure.outputs.runner-group }} | |
| benchmark-gcs-dir: ${{ steps.configure.outputs.benchmark-gcs-dir }} | |
| run-timestamp: ${{ steps.configure.outputs.run-timestamp }} | |
| steps: | |
| - name: "Checking out PR repository" | |
| uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 | |
| - name: "Configuring CI options" | |
| id: configure | |
| env: | |
| RUNNER_GROUP: ${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }} | |
| run: | | |
| # Just informative logging. There should only be two commits in the | |
| # history here, but limiting the depth helps when copying from a local | |
| # repo instead of using checkout, e.g. with | |
| # https://github.com/nektos/act where there will be more. | |
| git log --oneline --graph --max-count=3 | |
| # Workflow jobs can't access `env` in `runs-on`, so we need to make | |
| # `runner-group` a job output variable. | |
| echo "runner-group=${RUNNER_GROUP}" > "${GITHUB_OUTPUT}" | |
| RUN_TIMESTAMP="$(date +'%Y-%m-%d').$(date +'%s')" | |
| echo "run-timestamp=${RUN_TIMESTAMP}" >> "${GITHUB_OUTPUT}" | |
| # For presubmit testing, the result artifacts are uploaded to the | |
| # temporary workflow GCS dir. In postsubmit, the result artifacts are | |
| # uploaded to the comparative benchmark GCS dir. | |
| if [[ "${RUNNER_GROUP}" == "presubmit" ]]; then | |
| BENCHMARK_GCS_DIR="${GCS_DIR}/comparative-benchmark-artifacts" | |
| else | |
| BENCHMARK_GCS_DIR="gs://comparative-benchmark-artifacts/${RUN_TIMESTAMP}" | |
| fi | |
| echo "benchmark-gcs-dir=${BENCHMARK_GCS_DIR}" >> "${GITHUB_OUTPUT}" | |
| build_iree: | |
| needs: setup | |
| runs-on: | |
| - self-hosted # must come first | |
| - runner-group=${{ needs.setup.outputs.runner-group }} | |
| - environment=prod | |
| - cpu | |
| - os-family=Linux | |
| env: | |
| IREE_SOURCE_DIR: iree-src | |
| IREE_INSTALL_DIR: iree-install | |
| IREE_ANDROID_TOOLS_DIR: tools | |
| ANDROID_PLATFORM_VERSION: 31 | |
| X86_BUILD_DIR: iree-build | |
| ANDROID_BUILD_DIR: iree-build-android | |
| BENCHMARK_GCS_DIR: ${{ needs.setup.outputs.benchmark-gcs-dir }} | |
| outputs: | |
| iree-install-dir: ${{ env.IREE_INSTALL_DIR }} | |
| iree-install-dir-archive: ${{ steps.archive.outputs.iree-install-dir-archive }} | |
| iree-install-dir-gcs-artifact: ${{ steps.upload.outputs.iree-install-dir-gcs-artifact }} | |
| iree-android-tools-dir: ${{ env.IREE_ANDROID_TOOLS_DIR }} | |
| iree-android-tools-dir-archive: ${{ steps.archive.outputs.iree-android-tools-dir-archive }} | |
| iree-android-tools-dir-gcs-artifact: ${{ steps.upload.outputs.iree-android-tools-dir-gcs-artifact }} | |
| steps: | |
| - name: "Checking out PR repository" | |
| uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 | |
| - name: "Building IREE" | |
| run: | | |
| mkdir -p "${IREE_SOURCE_DIR}" | |
| mkdir -p "${X86_BUILD_DIR}" | |
| mkdir -p "${ANDROID_BUILD_DIR}" | |
| docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ | |
| --env IREE_SOURCE_DIR="${IREE_SOURCE_DIR}" \ | |
| --env IREE_INSTALL_DIR="${IREE_INSTALL_DIR}" \ | |
| --env ANDROID_PLATFORM_VERSION="${ANDROID_PLATFORM_VERSION}" \ | |
| "gcr.io/iree-oss/openxla-benchmark/android@sha256:cba9827bc47a6afb575efc72ed821ceda32b3132a406235d3edf6d125deef0d7" \ | |
| ./experimental/iree/build_iree.sh \ | |
| "${X86_BUILD_DIR}" \ | |
| "${ANDROID_BUILD_DIR}" | |
| - name: "Creating archives" | |
| id: archive | |
| env: | |
| IREE_INSTALL_DIR_ARCHIVE: ${{ env.IREE_INSTALL_DIR }}.tgz | |
| IREE_ANDROID_TOOLS_DIR_ARCHIVE: ${{ env.IREE_ANDROID_TOOLS_DIR }}.tgz | |
| run: | | |
| tar -zcvf "${IREE_INSTALL_DIR_ARCHIVE}" -C "${X86_BUILD_DIR}" "${IREE_INSTALL_DIR}" | |
| echo "iree-install-dir-archive=${IREE_INSTALL_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}" | |
| tar -zcvf "${IREE_ANDROID_TOOLS_DIR_ARCHIVE}" -C "${ANDROID_BUILD_DIR}" "${IREE_ANDROID_TOOLS_DIR}" | |
| echo "iree-android-tools-dir-archive=${IREE_ANDROID_TOOLS_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}" | |
| - name: "Uploading archives" | |
| id: upload | |
| env: | |
| IREE_INSTALL_DIR_ARCHIVE: ${{ steps.archive.outputs.iree-install-dir-archive }} | |
| IREE_INSTALL_DIR_GCS_ARTIFACT: ${{ env.BENCHMARK_GCS_DIR }}/${{ steps.archive.outputs.iree-install-dir-archive }} | |
| IREE_ANDROID_TOOLS_DIR_ARCHIVE: ${{ steps.archive.outputs.iree-android-tools-dir-archive }} | |
| IREE_ANDROID_TOOLS_DIR_GCS_ARTIFACT: ${{ env.BENCHMARK_GCS_DIR }}/${{ steps.archive.outputs.iree-android-tools-dir-archive }} | |
| run: | | |
| gcloud storage cp "${IREE_INSTALL_DIR_ARCHIVE}" "${IREE_INSTALL_DIR_GCS_ARTIFACT}" | |
| echo "iree-install-dir-gcs-artifact=${IREE_INSTALL_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}" | |
| gcloud storage cp "${IREE_ANDROID_TOOLS_DIR_ARCHIVE}" "${IREE_ANDROID_TOOLS_DIR_GCS_ARTIFACT}" | |
| echo "iree-android-tools-dir-gcs-artifact=${IREE_ANDROID_TOOLS_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}" | |
| build_tflite: | |
| needs: setup | |
| runs-on: | |
| - self-hosted # must come first | |
| - runner-group=${{ needs.setup.outputs.runner-group }} | |
| - environment=prod | |
| - cpu | |
| - os-family=Linux | |
| env: | |
| TF_SOURCE_DIR: tf-src | |
| X86_BUILD_DIR: tf-build | |
| ANDROID_BUILD_DIR: tf-build-android | |
| BENCHMARK_GCS_DIR: ${{ needs.setup.outputs.benchmark-gcs-dir }} | |
| outputs: | |
| tflite-build-dir: ${{ env.X86_BUILD_DIR }} | |
| tflite-build-dir-archive: ${{ steps.archive.outputs.tflite-build-dir-archive }} | |
| tflite-build-dir-gcs-artifact: ${{ steps.upload.outputs.tflite-build-dir-gcs-artifact }} | |
| tflite-android-build-dir: ${{ env.ANDROID_BUILD_DIR }} | |
| tflite-android-build-dir-archive: ${{ steps.archive.outputs.tflite-android-build-dir-archive }} | |
| tflite-android-build-dir-gcs-artifact: ${{ steps.upload.outputs.tflite-android-build-dir-gcs-artifact }} | |
| steps: | |
| - name: "Checking out PR repository" | |
| uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 | |
| - name: "Building TFLite" | |
| run: | | |
| mkdir -p "${TF_SOURCE_DIR}" | |
| mkdir -p "${X86_BUILD_DIR}" | |
| mkdir -p "${ANDROID_BUILD_DIR}" | |
| docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ | |
| --env OOBI_TEMP_DIR="${TF_SOURCE_DIR}" \ | |
| "gcr.io/iree-oss/openxla-benchmark/tflite-android@sha256:a9e6c363edb8a10d460dd2eebce22e59a005b5125104eb111ed6586470591cc2" \ | |
| ./experimental/tflite/build_tflite.sh \ | |
| "${X86_BUILD_DIR}" \ | |
| "${ANDROID_BUILD_DIR}" | |
| - name: "Creating archives" | |
| id: archive | |
| env: | |
| TFLITE_BUILD_DIR_ARCHIVE: ${{ env.X86_BUILD_DIR }}.tgz | |
| TFLITE_ANDROID_BUILD_DIR_ARCHIVE: ${{ env.ANDROID_BUILD_DIR }}.tgz | |
| run: | | |
| tar -zcvf "${TFLITE_BUILD_DIR_ARCHIVE}" "${X86_BUILD_DIR}" | |
| echo "tflite-build-dir-archive=${TFLITE_BUILD_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}" | |
| tar -zcvf "${TFLITE_ANDROID_BUILD_DIR_ARCHIVE}" "${ANDROID_BUILD_DIR}" | |
| echo "tflite-android-build-dir-archive=${TFLITE_ANDROID_BUILD_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}" | |
| - name: "Uploading archives" | |
| id: upload | |
| env: | |
| TFLITE_BUILD_DIR_ARCHIVE: ${{ steps.archive.outputs.tflite-build-dir-archive }} | |
| TFLITE_BUILD_DIR_GCS_ARTIFACT: ${{ env.BENCHMARK_GCS_DIR }}/${{ steps.archive.outputs.tflite-build-dir-archive }} | |
| TFLITE_ANDROID_BUILD_DIR_ARCHIVE: ${{ steps.archive.outputs.tflite-android-build-dir-archive }} | |
| TFLITE_ANDROID_BUILD_DIR_GCS_ARTIFACT: ${{ env.BENCHMARK_GCS_DIR }}/${{ steps.archive.outputs.tflite-android-build-dir-archive }} | |
| run: | | |
| gcloud storage cp "${TFLITE_BUILD_DIR_ARCHIVE}" "${TFLITE_BUILD_DIR_GCS_ARTIFACT}" | |
| echo "tflite-build-dir-gcs-artifact=${TFLITE_BUILD_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}" | |
| gcloud storage cp "${TFLITE_ANDROID_BUILD_DIR_ARCHIVE}" "${TFLITE_ANDROID_BUILD_DIR_GCS_ARTIFACT}" | |
| echo "tflite-android-build-dir-gcs-artifact=${TFLITE_ANDROID_BUILD_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}" | |
| # We compile and benchmark on the same machine because it takes too much time to compress and upload compiled artifacts. | |
| compile_and_benchmark_on_c2-standard-60: | |
| needs: [setup, build_iree, build_tflite] | |
| runs-on: | |
| - self-hosted # must come first | |
| - runner-group=${{ needs.setup.outputs.runner-group }} | |
| - environment=prod | |
| - machine-type=c2-standard-60 | |
| env: | |
| BENCHMARK_ARTIFACTS_DIR: benchmark-artifacts-dir | |
| TARGET_DEVICE: c2-standard-60 | |
| IREE_INSTALL_DIR: ${{ needs.build_iree.outputs.iree-install-dir }} | |
| IREE_INSTALL_DIR_ARCHIVE: ${{ needs.build_iree.outputs.iree-install-dir-archive }} | |
| IREE_INSTALL_DIR_GCS_ARTIFACT: ${{ needs.build_iree.outputs.iree-install-dir-gcs-artifact }} | |
| TFLITE_BUILD_DIR: ${{ needs.build_tflite.outputs.tflite-build-dir }} | |
| TFLITE_BUILD_DIR_ARCHIVE: ${{ needs.build_tflite.outputs.tflite-build-dir-archive }} | |
| TFLITE_BUILD_DIR_GCS_ARTIFACT: ${{ needs.build_tflite.outputs.tflite-build-dir-gcs-artifact }} | |
| BENCHMARK_GCS_DIR: ${{ needs.setup.outputs.benchmark-gcs-dir }} | |
| steps: | |
| - name: "Checking out PR repository" | |
| uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 | |
| - name: "Downloading and unpacking IREE tools" | |
| run: | | |
| gcloud storage cp "${IREE_INSTALL_DIR_GCS_ARTIFACT}" "${IREE_INSTALL_DIR_ARCHIVE}" | |
| tar -xvf "${IREE_INSTALL_DIR_ARCHIVE}" | |
| - name: "Compiling workloads" | |
| id: compile | |
| env: | |
| IREE_COMPILE_PATH: ${{ env.IREE_INSTALL_DIR }}/bin/iree-compile | |
| IREE_TEMP_DIR: iree-temp | |
| run: | | |
| mkdir "${BENCHMARK_ARTIFACTS_DIR}" | |
| mkdir "${IREE_TEMP_DIR}" | |
| docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ | |
| --env IREE_COMPILE_PATH="${IREE_COMPILE_PATH}" \ | |
| --env OOBI_TEMP_DIR="${IREE_TEMP_DIR}" \ | |
| "gcr.io/iree-oss/openxla-benchmark/base-python3.11@sha256:b9b98da7bcc5e431800ff798a6dcc394b1838a9ed3d695f5cd0dac3510fc8c8d" \ | |
| ./experimental/iree/compile_workloads.sh \ | |
| "${TARGET_DEVICE}" \ | |
| "${BENCHMARK_ARTIFACTS_DIR}" | |
| - name: "Benchmarking IREE:CPU" | |
| id: benchmark-iree | |
| env: | |
| IREE_RESULTS_FILE: iree-${{ env.TARGET_DEVICE }}.json | |
| RESULTS_GCS_DIR: ${{ env.BENCHMARK_GCS_DIR }}/${{ env.TARGET_DEVICE }}-results | |
| run: | | |
| docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ | |
| --env OOBI_IREE_BENCHMARK_MODULE_PATH="${IREE_INSTALL_DIR}/bin/iree-benchmark-module" \ | |
| --env OOBI_IREE_RUN_MODULE_PATH="${IREE_INSTALL_DIR}/bin/iree-run-module" \ | |
| "gcr.io/iree-oss/openxla-benchmark/base-python3.11@sha256:b9b98da7bcc5e431800ff798a6dcc394b1838a9ed3d695f5cd0dac3510fc8c8d" \ | |
| ./experimental/iree/benchmark_iree_x86.sh \ | |
| "${TARGET_DEVICE}" \ | |
| "${BENCHMARK_ARTIFACTS_DIR}" \ | |
| "${IREE_RESULTS_FILE}" | |
| gcloud storage cp "${IREE_RESULTS_FILE}" "${RESULTS_GCS_DIR}/" | |
| echo "iree-results-file=${IREE_RESULTS_FILE}" >> "${GITHUB_OUTPUT}" | |
| echo "iree-results-gcs-path=${RESULTS_GCS_DIR}/${IREE_RESULTS_FILE}" >> "${GITHUB_OUTPUT}" | |
| - name: "Downloading and unpacking TFLite tools" | |
| run: | | |
| gcloud storage cp "${TFLITE_BUILD_DIR_GCS_ARTIFACT}" "${TFLITE_BUILD_DIR_ARCHIVE}" | |
| tar -xvf "${TFLITE_BUILD_DIR_ARCHIVE}" | |
| - name: "Benchmarking TFLite:CPU" | |
| id: benchmark-tflite | |
| env: | |
| TFLITE_TEMP_DIR: tflite-temp | |
| TFLITE_RESULTS_FILE: tflite-${{ env.TARGET_DEVICE }}.json | |
| RESULTS_GCS_DIR: ${{ env.BENCHMARK_GCS_DIR }}/${{ env.TARGET_DEVICE }}-results | |
| run: | | |
| mkdir "${TFLITE_TEMP_DIR}" | |
| docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ | |
| --env OOBI_TEMP_DIR="${TFLITE_TEMP_DIR}" \ | |
| --env TFLITE_BENCHMARK_BINARY="${TFLITE_BUILD_DIR}/benchmark_model" \ | |
| "gcr.io/iree-oss/openxla-benchmark/base-python3.11@sha256:b9b98da7bcc5e431800ff798a6dcc394b1838a9ed3d695f5cd0dac3510fc8c8d" \ | |
| ./experimental/tflite/benchmark_tflite_x86.sh \ | |
| "${TARGET_DEVICE}" \ | |
| "${TFLITE_RESULTS_FILE}" | |
| gcloud storage cp "${TFLITE_RESULTS_FILE}" "${RESULTS_GCS_DIR}/" | |
| echo "tflite-results-file=${TFLITE_RESULTS_FILE}" >> "${GITHUB_OUTPUT}" | |
| echo "tflite-results-gcs-path=${RESULTS_GCS_DIR}/${TFLITE_RESULTS_FILE}" >> "${GITHUB_OUTPUT}" | |
| - name: "Generating report" | |
| env: | |
| IREE_RESULTS_FILE: ${{ steps.benchmark-iree.outputs.iree-results-file }} | |
| IREE_RESULTS_GCS_PATH: ${{ steps.benchmark-iree.outputs.iree-results-gcs-path }} | |
| TFLITE_RESULTS_FILE: ${{ steps.benchmark-tflite.outputs.tflite-results-file }} | |
| TFLITE_RESULTS_GCS_PATH: ${{ steps.benchmark-tflite.outputs.tflite-results-gcs-path }} | |
| SUMMARY_HTML: server_summary.html | |
| SUMMARY_HTML_GCS_PATH: "gs://tflite-benchmark-artifacts/server" | |
| RUN_TIMESTAMP: ${{ needs.setup.outputs.run-timestamp }} | |
| run: | | |
| gcloud storage cp "${IREE_RESULTS_GCS_PATH}" "${IREE_RESULTS_FILE}" | |
| gcloud storage cp "${TFLITE_RESULTS_GCS_PATH}" "${TFLITE_RESULTS_FILE}" | |
| docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ | |
| "gcr.io/iree-oss/openxla-benchmark/base-python3.11@sha256:b9b98da7bcc5e431800ff798a6dcc394b1838a9ed3d695f5cd0dac3510fc8c8d" \ | |
| ./experimental/utils/generate_iree_vs_tflite_report.sh \ | |
| "${IREE_RESULTS_FILE}" \ | |
| "${TFLITE_RESULTS_FILE}" \ | |
| "${SUMMARY_HTML}" | |
| gcloud storage cp "${SUMMARY_HTML}" "${BENCHMARK_GCS_DIR}/${SUMMARY_HTML}" | |
| gcloud storage cp "${IREE_RESULTS_FILE}" "${SUMMARY_HTML_GCS_PATH}/latest/iree.json" | |
| gcloud storage cp "${TFLITE_RESULTS_FILE}" "${SUMMARY_HTML_GCS_PATH}/latest/tflite.json" | |
| gcloud storage cp "${SUMMARY_HTML}" "${SUMMARY_HTML_GCS_PATH}/latest/${SUMMARY_HTML}" | |
| gcloud storage cp "${IREE_RESULTS_FILE}" "${SUMMARY_HTML_GCS_PATH}/${RUN_TIMESTAMP}/iree.json" | |
| gcloud storage cp "${TFLITE_RESULTS_FILE}" "${SUMMARY_HTML_GCS_PATH}/${RUN_TIMESTAMP}/tflite.json" | |
| gcloud storage cp "${SUMMARY_HTML}" "${SUMMARY_HTML_GCS_PATH}/${RUN_TIMESTAMP}/${SUMMARY_HTML}" | |
| compile_models_for_android: | |
| needs: [setup, build_iree] | |
| runs-on: | |
| - self-hosted # must come first | |
| - runner-group=${{ needs.setup.outputs.runner-group }} | |
| - environment=prod | |
| - cpu | |
| - os-family=Linux | |
| env: | |
| BENCHMARK_ARTIFACTS_DIR: android-benchmark-artifacts-dir | |
| TARGET_DEVICE: pixel-8-pro | |
| IREE_INSTALL_DIR: ${{ needs.build_iree.outputs.iree-install-dir }} | |
| IREE_INSTALL_DIR_ARCHIVE: ${{ needs.build_iree.outputs.iree-install-dir-archive }} | |
| IREE_INSTALL_DIR_GCS_ARTIFACT: ${{ needs.build_iree.outputs.iree-install-dir-gcs-artifact }} | |
| BENCHMARK_GCS_DIR: ${{ needs.setup.outputs.benchmark-gcs-dir }} | |
| outputs: | |
| benchmark-artifacts-dir: ${{ env.BENCHMARK_ARTIFACTS_DIR }} | |
| benchmark-artifacts-dir-archive: ${{ steps.archive.outputs.benchmark-artifacts-dir-archive }} | |
| benchmark-artifacts-dir-gcs-artifact: ${{ steps.upload.outputs.benchmark-artifacts-dir-gcs-artifact }} | |
| steps: | |
| - name: "Checking out PR repository" | |
| uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 | |
| - name: "Downloading and unpacking IREE tools" | |
| run: | | |
| gcloud storage cp "${IREE_INSTALL_DIR_GCS_ARTIFACT}" "${IREE_INSTALL_DIR_ARCHIVE}" | |
| tar -xvf "${IREE_INSTALL_DIR_ARCHIVE}" | |
| - name: "Compiling workloads" | |
| id: compile | |
| env: | |
| IREE_COMPILE_PATH: ${{ env.IREE_INSTALL_DIR }}/bin/iree-compile | |
| OOBI_TEMP_DIR: temp | |
| run: | | |
| mkdir "${BENCHMARK_ARTIFACTS_DIR}" | |
| mkdir "${OOBI_TEMP_DIR}" | |
| docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ | |
| --env IREE_COMPILE_PATH="${IREE_COMPILE_PATH}" \ | |
| --env OOBI_TEMP_DIR="${OOBI_TEMP_DIR}" \ | |
| "gcr.io/iree-oss/openxla-benchmark/android@sha256:cba9827bc47a6afb575efc72ed821ceda32b3132a406235d3edf6d125deef0d7" \ | |
| ./experimental/iree/compile_workloads.sh \ | |
| "${TARGET_DEVICE}" \ | |
| "${BENCHMARK_ARTIFACTS_DIR}" | |
| - name: "Creating archives" | |
| id: archive | |
| env: | |
| BENCHMARK_ARTIFACTS_DIR_ARCHIVE: ${{ env.BENCHMARK_ARTIFACTS_DIR }}.tgz | |
| run: | | |
| tar -cvf "${BENCHMARK_ARTIFACTS_DIR_ARCHIVE}" "${BENCHMARK_ARTIFACTS_DIR}" | |
| echo "benchmark-artifacts-dir-archive=${BENCHMARK_ARTIFACTS_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}" | |
| - name: "Uploading archives" | |
| id: upload | |
| env: | |
| BENCHMARK_ARTIFACTS_DIR_ARCHIVE: ${{ steps.archive.outputs.benchmark-artifacts-dir-archive }} | |
| BENCHMARK_ARTIFACTS_DIR_GCS_ARTIFACT: ${{ env.BENCHMARK_GCS_DIR }}/${{ steps.archive.outputs.benchmark-artifacts-dir-archive }} | |
| run: | | |
| gcloud storage cp "${BENCHMARK_ARTIFACTS_DIR_ARCHIVE}" "${BENCHMARK_ARTIFACTS_DIR_GCS_ARTIFACT}" | |
| echo "benchmark-artifacts-dir-gcs-artifact=${BENCHMARK_ARTIFACTS_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}" | |
| benchmark_on_pixel-8-pro: | |
| needs: [setup, build_iree, compile_models_for_android, build_tflite] | |
| runs-on: | |
| - self-hosted # must come first | |
| - runner-group=${{ needs.setup.outputs.runner-group }} | |
| - environment=prod | |
| - machine-type=pixel-8-pro | |
| env: | |
| BENCHMARK_GCS_DIR: ${{ needs.setup.outputs.benchmark-gcs-dir }} | |
| RESULTS_DIR: results-dir | |
| TARGET_DEVICE: pixel-8-pro | |
| IREE_ANDROID_TOOLS_DIR: ${{ needs.build_iree.outputs.iree-android-tools-dir }} | |
| IREE_ANDROID_TOOLS_DIR_ARCHIVE: ${{ needs.build_iree.outputs.iree-android-tools-dir-archive }} | |
| IREE_ANDROID_TOOLS_DIR_GCS_ARTIFACT: ${{ needs.build_iree.outputs.iree-android-tools-dir-gcs-artifact }} | |
| IREE_ARTIFACTS_DIR: ${{ needs.compile_models_for_android.outputs.benchmark-artifacts-dir }} | |
| IREE_ARTIFACTS_DIR_ARCHIVE: ${{ needs.compile_models_for_android.outputs.benchmark-artifacts-dir-archive }} | |
| IREE_ARTIFACTS_DIR_GCS_ARTIFACT: ${{ needs.compile_models_for_android.outputs.benchmark-artifacts-dir-gcs-artifact }} | |
| TFLITE_ANDROID_BUILD_DIR: ${{ needs.build_tflite.outputs.tflite-android-build-dir }} | |
| TFLITE_ANDROID_BUILD_DIR_ARCHIVE: ${{ needs.build_tflite.outputs.tflite-android-build-dir-archive }} | |
| TFLITE_ANDROID_BUILD_DIR_GCS_ARTIFACT: ${{ needs.build_tflite.outputs.tflite-android-build-dir-gcs-artifact }} | |
| steps: | |
| - name: "Checking out PR repository" | |
| uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 | |
| - name: "Setup" | |
| id: setup | |
| run: | | |
| echo "results-gcs-dir=${BENCHMARK_GCS_DIR}/${TARGET_DEVICE}-results" >> "${GITHUB_OUTPUT}" | |
| mkdir "${RESULTS_DIR}" | |
| - name: "Downloading and unpacking tools" | |
| run: | | |
| gcloud storage cp "${IREE_ANDROID_TOOLS_DIR_GCS_ARTIFACT}" "${IREE_ANDROID_TOOLS_DIR_ARCHIVE}" | |
| tar -xvf "${IREE_ANDROID_TOOLS_DIR_ARCHIVE}" | |
| - name: "Benchmarking IREE on Android" | |
| id: benchmark-iree | |
| env: | |
| IREE_RESULTS_FILE: iree-${{ env.TARGET_DEVICE }}.json | |
| RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} | |
| run: | | |
| RESULTS_PATH="${RESULTS_DIR}/${IREE_RESULTS_FILE}" | |
| # We use the `https` path instead of `gs` since artifact streaming only works with `https`. | |
| IREE_ARTIFACTS_DIR_HTTPS_ARTIFACT=$(echo "${IREE_ARTIFACTS_DIR_GCS_ARTIFACT}" | sed -E 's/gs:\/\/(.*)/https:\/\/storage.googleapis.com\/\1/') | |
| OOBI_IREE_RUN_MODULE_PATH="${IREE_ANDROID_TOOLS_DIR}/iree-run-module" \ | |
| OOBI_IREE_BENCHMARK_MODULE_PATH="${IREE_ANDROID_TOOLS_DIR}/iree-benchmark-module" \ | |
| ./experimental/iree/benchmark_iree_android.sh \ | |
| "${TARGET_DEVICE}" \ | |
| "${IREE_ARTIFACTS_DIR_HTTPS_ARTIFACT}" \ | |
| "${RESULTS_PATH}" | |
| gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" | |
| echo "iree-results-file=${IREE_RESULTS_FILE}" >> "${GITHUB_OUTPUT}" | |
| echo "iree-results-gcs-path=${RESULTS_GCS_DIR}/${IREE_RESULTS_FILE}" >> "${GITHUB_OUTPUT}" | |
| - name: "Downloading and unpacking TFLite tools" | |
| run: | | |
| gcloud storage cp "${TFLITE_ANDROID_BUILD_DIR_GCS_ARTIFACT}" "${TFLITE_ANDROID_BUILD_DIR_ARCHIVE}" | |
| tar -xvf "${TFLITE_ANDROID_BUILD_DIR_ARCHIVE}" | |
| - name: "Benchmarking TFLite on Android" | |
| id: benchmark-tflite | |
| env: | |
| TFLITE_RESULTS_FILE: tflite-${{ env.TARGET_DEVICE }}.json | |
| RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} | |
| run: | | |
| RESULTS_PATH="${RESULTS_DIR}/${TFLITE_RESULTS_FILE}" | |
| TFLITE_BENCHMARK_BINARY="${TFLITE_ANDROID_BUILD_DIR}/benchmark_model" \ | |
| ./experimental/tflite/benchmark_tflite_android.sh \ | |
| "${TARGET_DEVICE}" \ | |
| "${RESULTS_PATH}" | |
| gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" | |
| echo "tflite-results-file=${TFLITE_RESULTS_FILE}" >> "${GITHUB_OUTPUT}" | |
| echo "tflite-results-gcs-path=${RESULTS_GCS_DIR}/${TFLITE_RESULTS_FILE}" >> "${GITHUB_OUTPUT}" | |
| - name: "Generating report" | |
| env: | |
| IREE_RESULTS_FILE: ${{ steps.benchmark-iree.outputs.iree-results-file }} | |
| IREE_RESULTS_GCS_PATH: ${{ steps.benchmark-iree.outputs.iree-results-gcs-path }} | |
| TFLITE_RESULTS_FILE: ${{ steps.benchmark-tflite.outputs.tflite-results-file }} | |
| TFLITE_RESULTS_GCS_PATH: ${{ steps.benchmark-tflite.outputs.tflite-results-gcs-path }} | |
| SUMMARY_HTML: mobile_summary.html | |
| SUMMARY_HTML_GCS_PATH: "gs://tflite-benchmark-artifacts/mobile" | |
| BENCHMARK_GCS_DIR: ${{ needs.setup.outputs.benchmark-gcs-dir }} | |
| RUN_TIMESTAMP: ${{ needs.setup.outputs.run-timestamp }} | |
| run: | | |
| gcloud storage cp "${IREE_RESULTS_GCS_PATH}" "${IREE_RESULTS_FILE}" | |
| gcloud storage cp "${TFLITE_RESULTS_GCS_PATH}" "${TFLITE_RESULTS_FILE}" | |
| ./experimental/utils/generate_iree_vs_tflite_report.sh \ | |
| "${IREE_RESULTS_FILE}" \ | |
| "${TFLITE_RESULTS_FILE}" \ | |
| "${SUMMARY_HTML}" | |
| gcloud storage cp "${SUMMARY_HTML}" "${BENCHMARK_GCS_DIR}/${SUMMARY_HTML}" | |
| gcloud storage cp "${IREE_RESULTS_FILE}" "${SUMMARY_HTML_GCS_PATH}/latest/iree.json" | |
| gcloud storage cp "${TFLITE_RESULTS_FILE}" "${SUMMARY_HTML_GCS_PATH}/latest/tflite.json" | |
| gcloud storage cp "${SUMMARY_HTML}" "${SUMMARY_HTML_GCS_PATH}/latest/${SUMMARY_HTML}" | |
| gcloud storage cp "${IREE_RESULTS_FILE}" "${SUMMARY_HTML_GCS_PATH}/${RUN_TIMESTAMP}/iree.json" | |
| gcloud storage cp "${TFLITE_RESULTS_FILE}" "${SUMMARY_HTML_GCS_PATH}/${RUN_TIMESTAMP}/tflite.json" | |
| gcloud storage cp "${SUMMARY_HTML}" "${SUMMARY_HTML_GCS_PATH}/${RUN_TIMESTAMP}/${SUMMARY_HTML}" |