Skip to content

Mobile Comparative Benchmarks #665

Mobile Comparative Benchmarks

Mobile Comparative Benchmarks #665

# Copyright 2023 The OpenXLA Authors
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
# Mobile Comparative Benchmarks Workflow.
name: Mobile Comparative Benchmarks
on:
workflow_dispatch:
schedule:
# Scheduled to run at 09:00 UTC and 21:00 UTC.
- cron: '0 09,21 * * *'
concurrency:
# A PR number if a pull request and otherwise the commit hash. This cancels
# queued and in-progress runs for the same PR (presubmit) or commit
# (postsubmit).
group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
cancel-in-progress: true
env:
GCS_DIR: gs://openxla-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }}
jobs:
setup:
runs-on: ubuntu-22.04
outputs:
runner-group: ${{ steps.configure.outputs.runner-group }}
benchmark-gcs-dir: ${{ steps.configure.outputs.benchmark-gcs-dir }}
run-timestamp: ${{ steps.configure.outputs.run-timestamp }}
steps:
- name: "Checking out PR repository"
uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0
- name: "Configuring CI options"
id: configure
env:
RUNNER_GROUP: ${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
run: |
# Just informative logging. There should only be two commits in the
# history here, but limiting the depth helps when copying from a local
# repo instead of using checkout, e.g. with
# https://github.com/nektos/act where there will be more.
git log --oneline --graph --max-count=3
# Workflow jobs can't access `env` in `runs-on`, so we need to make
# `runner-group` a job output variable.
echo "runner-group=${RUNNER_GROUP}" > "${GITHUB_OUTPUT}"
RUN_TIMESTAMP="$(date +'%Y-%m-%d').$(date +'%s')"
echo "run-timestamp=${RUN_TIMESTAMP}" >> "${GITHUB_OUTPUT}"
# For presubmit testing, the result artifacts are uploaded to the
# temporary workflow GCS dir. In postsubmit, the result artifacts are
# uploaded to the comparative benchmark GCS dir.
if [[ "${RUNNER_GROUP}" == "presubmit" ]]; then
BENCHMARK_GCS_DIR="${GCS_DIR}/comparative-benchmark-artifacts"
else
BENCHMARK_GCS_DIR="gs://comparative-benchmark-artifacts/${RUN_TIMESTAMP}"
fi
echo "benchmark-gcs-dir=${BENCHMARK_GCS_DIR}" >> "${GITHUB_OUTPUT}"
build_iree:
needs: setup
runs-on:
- self-hosted # must come first
- runner-group=${{ needs.setup.outputs.runner-group }}
- environment=prod
- cpu
- os-family=Linux
env:
IREE_SOURCE_DIR: iree-src
IREE_INSTALL_DIR: iree-install
IREE_ANDROID_TOOLS_DIR: tools
ANDROID_PLATFORM_VERSION: 31
X86_BUILD_DIR: iree-build
ANDROID_BUILD_DIR: iree-build-android
BENCHMARK_GCS_DIR: ${{ needs.setup.outputs.benchmark-gcs-dir }}
outputs:
iree-install-dir: ${{ env.IREE_INSTALL_DIR }}
iree-install-dir-archive: ${{ steps.archive.outputs.iree-install-dir-archive }}
iree-install-dir-gcs-artifact: ${{ steps.upload.outputs.iree-install-dir-gcs-artifact }}
iree-android-tools-dir: ${{ env.IREE_ANDROID_TOOLS_DIR }}
iree-android-tools-dir-archive: ${{ steps.archive.outputs.iree-android-tools-dir-archive }}
iree-android-tools-dir-gcs-artifact: ${{ steps.upload.outputs.iree-android-tools-dir-gcs-artifact }}
steps:
- name: "Checking out PR repository"
uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0
- name: "Building IREE"
run: |
mkdir -p "${IREE_SOURCE_DIR}"
mkdir -p "${X86_BUILD_DIR}"
mkdir -p "${ANDROID_BUILD_DIR}"
docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \
--env IREE_SOURCE_DIR="${IREE_SOURCE_DIR}" \
--env IREE_INSTALL_DIR="${IREE_INSTALL_DIR}" \
--env ANDROID_PLATFORM_VERSION="${ANDROID_PLATFORM_VERSION}" \
"gcr.io/iree-oss/openxla-benchmark/android@sha256:cba9827bc47a6afb575efc72ed821ceda32b3132a406235d3edf6d125deef0d7" \
./experimental/iree/build_iree.sh \
"${X86_BUILD_DIR}" \
"${ANDROID_BUILD_DIR}"
- name: "Creating archives"
id: archive
env:
IREE_INSTALL_DIR_ARCHIVE: ${{ env.IREE_INSTALL_DIR }}.tgz
IREE_ANDROID_TOOLS_DIR_ARCHIVE: ${{ env.IREE_ANDROID_TOOLS_DIR }}.tgz
run: |
tar -zcvf "${IREE_INSTALL_DIR_ARCHIVE}" -C "${X86_BUILD_DIR}" "${IREE_INSTALL_DIR}"
echo "iree-install-dir-archive=${IREE_INSTALL_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}"
tar -zcvf "${IREE_ANDROID_TOOLS_DIR_ARCHIVE}" -C "${ANDROID_BUILD_DIR}" "${IREE_ANDROID_TOOLS_DIR}"
echo "iree-android-tools-dir-archive=${IREE_ANDROID_TOOLS_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}"
- name: "Uploading archives"
id: upload
env:
IREE_INSTALL_DIR_ARCHIVE: ${{ steps.archive.outputs.iree-install-dir-archive }}
IREE_INSTALL_DIR_GCS_ARTIFACT: ${{ env.BENCHMARK_GCS_DIR }}/${{ steps.archive.outputs.iree-install-dir-archive }}
IREE_ANDROID_TOOLS_DIR_ARCHIVE: ${{ steps.archive.outputs.iree-android-tools-dir-archive }}
IREE_ANDROID_TOOLS_DIR_GCS_ARTIFACT: ${{ env.BENCHMARK_GCS_DIR }}/${{ steps.archive.outputs.iree-android-tools-dir-archive }}
run: |
gcloud storage cp "${IREE_INSTALL_DIR_ARCHIVE}" "${IREE_INSTALL_DIR_GCS_ARTIFACT}"
echo "iree-install-dir-gcs-artifact=${IREE_INSTALL_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}"
gcloud storage cp "${IREE_ANDROID_TOOLS_DIR_ARCHIVE}" "${IREE_ANDROID_TOOLS_DIR_GCS_ARTIFACT}"
echo "iree-android-tools-dir-gcs-artifact=${IREE_ANDROID_TOOLS_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}"
build_tflite:
needs: setup
runs-on:
- self-hosted # must come first
- runner-group=${{ needs.setup.outputs.runner-group }}
- environment=prod
- cpu
- os-family=Linux
env:
TF_SOURCE_DIR: tf-src
X86_BUILD_DIR: tf-build
ANDROID_BUILD_DIR: tf-build-android
BENCHMARK_GCS_DIR: ${{ needs.setup.outputs.benchmark-gcs-dir }}
outputs:
tflite-build-dir: ${{ env.X86_BUILD_DIR }}
tflite-build-dir-archive: ${{ steps.archive.outputs.tflite-build-dir-archive }}
tflite-build-dir-gcs-artifact: ${{ steps.upload.outputs.tflite-build-dir-gcs-artifact }}
tflite-android-build-dir: ${{ env.ANDROID_BUILD_DIR }}
tflite-android-build-dir-archive: ${{ steps.archive.outputs.tflite-android-build-dir-archive }}
tflite-android-build-dir-gcs-artifact: ${{ steps.upload.outputs.tflite-android-build-dir-gcs-artifact }}
steps:
- name: "Checking out PR repository"
uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0
- name: "Building TFLite"
run: |
mkdir -p "${TF_SOURCE_DIR}"
mkdir -p "${X86_BUILD_DIR}"
mkdir -p "${ANDROID_BUILD_DIR}"
docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \
--env OOBI_TEMP_DIR="${TF_SOURCE_DIR}" \
"gcr.io/iree-oss/openxla-benchmark/tflite-android@sha256:a9e6c363edb8a10d460dd2eebce22e59a005b5125104eb111ed6586470591cc2" \
./experimental/tflite/build_tflite.sh \
"${X86_BUILD_DIR}" \
"${ANDROID_BUILD_DIR}"
- name: "Creating archives"
id: archive
env:
TFLITE_BUILD_DIR_ARCHIVE: ${{ env.X86_BUILD_DIR }}.tgz
TFLITE_ANDROID_BUILD_DIR_ARCHIVE: ${{ env.ANDROID_BUILD_DIR }}.tgz
run: |
tar -zcvf "${TFLITE_BUILD_DIR_ARCHIVE}" "${X86_BUILD_DIR}"
echo "tflite-build-dir-archive=${TFLITE_BUILD_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}"
tar -zcvf "${TFLITE_ANDROID_BUILD_DIR_ARCHIVE}" "${ANDROID_BUILD_DIR}"
echo "tflite-android-build-dir-archive=${TFLITE_ANDROID_BUILD_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}"
- name: "Uploading archives"
id: upload
env:
TFLITE_BUILD_DIR_ARCHIVE: ${{ steps.archive.outputs.tflite-build-dir-archive }}
TFLITE_BUILD_DIR_GCS_ARTIFACT: ${{ env.BENCHMARK_GCS_DIR }}/${{ steps.archive.outputs.tflite-build-dir-archive }}
TFLITE_ANDROID_BUILD_DIR_ARCHIVE: ${{ steps.archive.outputs.tflite-android-build-dir-archive }}
TFLITE_ANDROID_BUILD_DIR_GCS_ARTIFACT: ${{ env.BENCHMARK_GCS_DIR }}/${{ steps.archive.outputs.tflite-android-build-dir-archive }}
run: |
gcloud storage cp "${TFLITE_BUILD_DIR_ARCHIVE}" "${TFLITE_BUILD_DIR_GCS_ARTIFACT}"
echo "tflite-build-dir-gcs-artifact=${TFLITE_BUILD_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}"
gcloud storage cp "${TFLITE_ANDROID_BUILD_DIR_ARCHIVE}" "${TFLITE_ANDROID_BUILD_DIR_GCS_ARTIFACT}"
echo "tflite-android-build-dir-gcs-artifact=${TFLITE_ANDROID_BUILD_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}"
# We compile and benchmark on the same machine because it takes too much time to compress and upload compiled artifacts.
compile_and_benchmark_on_c2-standard-60:
needs: [setup, build_iree, build_tflite]
runs-on:
- self-hosted # must come first
- runner-group=${{ needs.setup.outputs.runner-group }}
- environment=prod
- machine-type=c2-standard-60
env:
BENCHMARK_ARTIFACTS_DIR: benchmark-artifacts-dir
TARGET_DEVICE: c2-standard-60
IREE_INSTALL_DIR: ${{ needs.build_iree.outputs.iree-install-dir }}
IREE_INSTALL_DIR_ARCHIVE: ${{ needs.build_iree.outputs.iree-install-dir-archive }}
IREE_INSTALL_DIR_GCS_ARTIFACT: ${{ needs.build_iree.outputs.iree-install-dir-gcs-artifact }}
TFLITE_BUILD_DIR: ${{ needs.build_tflite.outputs.tflite-build-dir }}
TFLITE_BUILD_DIR_ARCHIVE: ${{ needs.build_tflite.outputs.tflite-build-dir-archive }}
TFLITE_BUILD_DIR_GCS_ARTIFACT: ${{ needs.build_tflite.outputs.tflite-build-dir-gcs-artifact }}
BENCHMARK_GCS_DIR: ${{ needs.setup.outputs.benchmark-gcs-dir }}
steps:
- name: "Checking out PR repository"
uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0
- name: "Downloading and unpacking IREE tools"
run: |
gcloud storage cp "${IREE_INSTALL_DIR_GCS_ARTIFACT}" "${IREE_INSTALL_DIR_ARCHIVE}"
tar -xvf "${IREE_INSTALL_DIR_ARCHIVE}"
- name: "Compiling workloads"
id: compile
env:
IREE_COMPILE_PATH: ${{ env.IREE_INSTALL_DIR }}/bin/iree-compile
IREE_TEMP_DIR: iree-temp
run: |
mkdir "${BENCHMARK_ARTIFACTS_DIR}"
mkdir "${IREE_TEMP_DIR}"
docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \
--env IREE_COMPILE_PATH="${IREE_COMPILE_PATH}" \
--env OOBI_TEMP_DIR="${IREE_TEMP_DIR}" \
"gcr.io/iree-oss/openxla-benchmark/base-python3.11@sha256:b9b98da7bcc5e431800ff798a6dcc394b1838a9ed3d695f5cd0dac3510fc8c8d" \
./experimental/iree/compile_workloads.sh \
"${TARGET_DEVICE}" \
"${BENCHMARK_ARTIFACTS_DIR}"
- name: "Benchmarking IREE:CPU"
id: benchmark-iree
env:
IREE_RESULTS_FILE: iree-${{ env.TARGET_DEVICE }}.json
RESULTS_GCS_DIR: ${{ env.BENCHMARK_GCS_DIR }}/${{ env.TARGET_DEVICE }}-results
run: |
docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \
--env OOBI_IREE_BENCHMARK_MODULE_PATH="${IREE_INSTALL_DIR}/bin/iree-benchmark-module" \
--env OOBI_IREE_RUN_MODULE_PATH="${IREE_INSTALL_DIR}/bin/iree-run-module" \
"gcr.io/iree-oss/openxla-benchmark/base-python3.11@sha256:b9b98da7bcc5e431800ff798a6dcc394b1838a9ed3d695f5cd0dac3510fc8c8d" \
./experimental/iree/benchmark_iree_x86.sh \
"${TARGET_DEVICE}" \
"${BENCHMARK_ARTIFACTS_DIR}" \
"${IREE_RESULTS_FILE}"
gcloud storage cp "${IREE_RESULTS_FILE}" "${RESULTS_GCS_DIR}/"
echo "iree-results-file=${IREE_RESULTS_FILE}" >> "${GITHUB_OUTPUT}"
echo "iree-results-gcs-path=${RESULTS_GCS_DIR}/${IREE_RESULTS_FILE}" >> "${GITHUB_OUTPUT}"
- name: "Downloading and unpacking TFLite tools"
run: |
gcloud storage cp "${TFLITE_BUILD_DIR_GCS_ARTIFACT}" "${TFLITE_BUILD_DIR_ARCHIVE}"
tar -xvf "${TFLITE_BUILD_DIR_ARCHIVE}"
- name: "Benchmarking TFLite:CPU"
id: benchmark-tflite
env:
TFLITE_TEMP_DIR: tflite-temp
TFLITE_RESULTS_FILE: tflite-${{ env.TARGET_DEVICE }}.json
RESULTS_GCS_DIR: ${{ env.BENCHMARK_GCS_DIR }}/${{ env.TARGET_DEVICE }}-results
run: |
mkdir "${TFLITE_TEMP_DIR}"
docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \
--env OOBI_TEMP_DIR="${TFLITE_TEMP_DIR}" \
--env TFLITE_BENCHMARK_BINARY="${TFLITE_BUILD_DIR}/benchmark_model" \
"gcr.io/iree-oss/openxla-benchmark/base-python3.11@sha256:b9b98da7bcc5e431800ff798a6dcc394b1838a9ed3d695f5cd0dac3510fc8c8d" \
./experimental/tflite/benchmark_tflite_x86.sh \
"${TARGET_DEVICE}" \
"${TFLITE_RESULTS_FILE}"
gcloud storage cp "${TFLITE_RESULTS_FILE}" "${RESULTS_GCS_DIR}/"
echo "tflite-results-file=${TFLITE_RESULTS_FILE}" >> "${GITHUB_OUTPUT}"
echo "tflite-results-gcs-path=${RESULTS_GCS_DIR}/${TFLITE_RESULTS_FILE}" >> "${GITHUB_OUTPUT}"
- name: "Generating report"
env:
IREE_RESULTS_FILE: ${{ steps.benchmark-iree.outputs.iree-results-file }}
IREE_RESULTS_GCS_PATH: ${{ steps.benchmark-iree.outputs.iree-results-gcs-path }}
TFLITE_RESULTS_FILE: ${{ steps.benchmark-tflite.outputs.tflite-results-file }}
TFLITE_RESULTS_GCS_PATH: ${{ steps.benchmark-tflite.outputs.tflite-results-gcs-path }}
SUMMARY_HTML: server_summary.html
SUMMARY_HTML_GCS_PATH: "gs://tflite-benchmark-artifacts/server"
RUN_TIMESTAMP: ${{ needs.setup.outputs.run-timestamp }}
run: |
gcloud storage cp "${IREE_RESULTS_GCS_PATH}" "${IREE_RESULTS_FILE}"
gcloud storage cp "${TFLITE_RESULTS_GCS_PATH}" "${TFLITE_RESULTS_FILE}"
docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \
"gcr.io/iree-oss/openxla-benchmark/base-python3.11@sha256:b9b98da7bcc5e431800ff798a6dcc394b1838a9ed3d695f5cd0dac3510fc8c8d" \
./experimental/utils/generate_iree_vs_tflite_report.sh \
"${IREE_RESULTS_FILE}" \
"${TFLITE_RESULTS_FILE}" \
"${SUMMARY_HTML}"
gcloud storage cp "${SUMMARY_HTML}" "${BENCHMARK_GCS_DIR}/${SUMMARY_HTML}"
gcloud storage cp "${IREE_RESULTS_FILE}" "${SUMMARY_HTML_GCS_PATH}/latest/iree.json"
gcloud storage cp "${TFLITE_RESULTS_FILE}" "${SUMMARY_HTML_GCS_PATH}/latest/tflite.json"
gcloud storage cp "${SUMMARY_HTML}" "${SUMMARY_HTML_GCS_PATH}/latest/${SUMMARY_HTML}"
gcloud storage cp "${IREE_RESULTS_FILE}" "${SUMMARY_HTML_GCS_PATH}/${RUN_TIMESTAMP}/iree.json"
gcloud storage cp "${TFLITE_RESULTS_FILE}" "${SUMMARY_HTML_GCS_PATH}/${RUN_TIMESTAMP}/tflite.json"
gcloud storage cp "${SUMMARY_HTML}" "${SUMMARY_HTML_GCS_PATH}/${RUN_TIMESTAMP}/${SUMMARY_HTML}"
compile_models_for_android:
needs: [setup, build_iree]
runs-on:
- self-hosted # must come first
- runner-group=${{ needs.setup.outputs.runner-group }}
- environment=prod
- cpu
- os-family=Linux
env:
BENCHMARK_ARTIFACTS_DIR: android-benchmark-artifacts-dir
TARGET_DEVICE: pixel-8-pro
IREE_INSTALL_DIR: ${{ needs.build_iree.outputs.iree-install-dir }}
IREE_INSTALL_DIR_ARCHIVE: ${{ needs.build_iree.outputs.iree-install-dir-archive }}
IREE_INSTALL_DIR_GCS_ARTIFACT: ${{ needs.build_iree.outputs.iree-install-dir-gcs-artifact }}
BENCHMARK_GCS_DIR: ${{ needs.setup.outputs.benchmark-gcs-dir }}
outputs:
benchmark-artifacts-dir: ${{ env.BENCHMARK_ARTIFACTS_DIR }}
benchmark-artifacts-dir-archive: ${{ steps.archive.outputs.benchmark-artifacts-dir-archive }}
benchmark-artifacts-dir-gcs-artifact: ${{ steps.upload.outputs.benchmark-artifacts-dir-gcs-artifact }}
steps:
- name: "Checking out PR repository"
uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0
- name: "Downloading and unpacking IREE tools"
run: |
gcloud storage cp "${IREE_INSTALL_DIR_GCS_ARTIFACT}" "${IREE_INSTALL_DIR_ARCHIVE}"
tar -xvf "${IREE_INSTALL_DIR_ARCHIVE}"
- name: "Compiling workloads"
id: compile
env:
IREE_COMPILE_PATH: ${{ env.IREE_INSTALL_DIR }}/bin/iree-compile
OOBI_TEMP_DIR: temp
run: |
mkdir "${BENCHMARK_ARTIFACTS_DIR}"
mkdir "${OOBI_TEMP_DIR}"
docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \
--env IREE_COMPILE_PATH="${IREE_COMPILE_PATH}" \
--env OOBI_TEMP_DIR="${OOBI_TEMP_DIR}" \
"gcr.io/iree-oss/openxla-benchmark/android@sha256:cba9827bc47a6afb575efc72ed821ceda32b3132a406235d3edf6d125deef0d7" \
./experimental/iree/compile_workloads.sh \
"${TARGET_DEVICE}" \
"${BENCHMARK_ARTIFACTS_DIR}"
- name: "Creating archives"
id: archive
env:
BENCHMARK_ARTIFACTS_DIR_ARCHIVE: ${{ env.BENCHMARK_ARTIFACTS_DIR }}.tgz
run: |
tar -cvf "${BENCHMARK_ARTIFACTS_DIR_ARCHIVE}" "${BENCHMARK_ARTIFACTS_DIR}"
echo "benchmark-artifacts-dir-archive=${BENCHMARK_ARTIFACTS_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}"
- name: "Uploading archives"
id: upload
env:
BENCHMARK_ARTIFACTS_DIR_ARCHIVE: ${{ steps.archive.outputs.benchmark-artifacts-dir-archive }}
BENCHMARK_ARTIFACTS_DIR_GCS_ARTIFACT: ${{ env.BENCHMARK_GCS_DIR }}/${{ steps.archive.outputs.benchmark-artifacts-dir-archive }}
run: |
gcloud storage cp "${BENCHMARK_ARTIFACTS_DIR_ARCHIVE}" "${BENCHMARK_ARTIFACTS_DIR_GCS_ARTIFACT}"
echo "benchmark-artifacts-dir-gcs-artifact=${BENCHMARK_ARTIFACTS_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}"
benchmark_on_pixel-8-pro:
needs: [setup, build_iree, compile_models_for_android, build_tflite]
runs-on:
- self-hosted # must come first
- runner-group=${{ needs.setup.outputs.runner-group }}
- environment=prod
- machine-type=pixel-8-pro
env:
BENCHMARK_GCS_DIR: ${{ needs.setup.outputs.benchmark-gcs-dir }}
RESULTS_DIR: results-dir
TARGET_DEVICE: pixel-8-pro
IREE_ANDROID_TOOLS_DIR: ${{ needs.build_iree.outputs.iree-android-tools-dir }}
IREE_ANDROID_TOOLS_DIR_ARCHIVE: ${{ needs.build_iree.outputs.iree-android-tools-dir-archive }}
IREE_ANDROID_TOOLS_DIR_GCS_ARTIFACT: ${{ needs.build_iree.outputs.iree-android-tools-dir-gcs-artifact }}
IREE_ARTIFACTS_DIR: ${{ needs.compile_models_for_android.outputs.benchmark-artifacts-dir }}
IREE_ARTIFACTS_DIR_ARCHIVE: ${{ needs.compile_models_for_android.outputs.benchmark-artifacts-dir-archive }}
IREE_ARTIFACTS_DIR_GCS_ARTIFACT: ${{ needs.compile_models_for_android.outputs.benchmark-artifacts-dir-gcs-artifact }}
TFLITE_ANDROID_BUILD_DIR: ${{ needs.build_tflite.outputs.tflite-android-build-dir }}
TFLITE_ANDROID_BUILD_DIR_ARCHIVE: ${{ needs.build_tflite.outputs.tflite-android-build-dir-archive }}
TFLITE_ANDROID_BUILD_DIR_GCS_ARTIFACT: ${{ needs.build_tflite.outputs.tflite-android-build-dir-gcs-artifact }}
steps:
- name: "Checking out PR repository"
uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0
- name: "Setup"
id: setup
run: |
echo "results-gcs-dir=${BENCHMARK_GCS_DIR}/${TARGET_DEVICE}-results" >> "${GITHUB_OUTPUT}"
mkdir "${RESULTS_DIR}"
- name: "Downloading and unpacking tools"
run: |
gcloud storage cp "${IREE_ANDROID_TOOLS_DIR_GCS_ARTIFACT}" "${IREE_ANDROID_TOOLS_DIR_ARCHIVE}"
tar -xvf "${IREE_ANDROID_TOOLS_DIR_ARCHIVE}"
- name: "Benchmarking IREE on Android"
id: benchmark-iree
env:
IREE_RESULTS_FILE: iree-${{ env.TARGET_DEVICE }}.json
RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }}
run: |
RESULTS_PATH="${RESULTS_DIR}/${IREE_RESULTS_FILE}"
# We use the `https` path instead of `gs` since artifact streaming only works with `https`.
IREE_ARTIFACTS_DIR_HTTPS_ARTIFACT=$(echo "${IREE_ARTIFACTS_DIR_GCS_ARTIFACT}" | sed -E 's/gs:\/\/(.*)/https:\/\/storage.googleapis.com\/\1/')
OOBI_IREE_RUN_MODULE_PATH="${IREE_ANDROID_TOOLS_DIR}/iree-run-module" \
OOBI_IREE_BENCHMARK_MODULE_PATH="${IREE_ANDROID_TOOLS_DIR}/iree-benchmark-module" \
./experimental/iree/benchmark_iree_android.sh \
"${TARGET_DEVICE}" \
"${IREE_ARTIFACTS_DIR_HTTPS_ARTIFACT}" \
"${RESULTS_PATH}"
gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/"
echo "iree-results-file=${IREE_RESULTS_FILE}" >> "${GITHUB_OUTPUT}"
echo "iree-results-gcs-path=${RESULTS_GCS_DIR}/${IREE_RESULTS_FILE}" >> "${GITHUB_OUTPUT}"
- name: "Downloading and unpacking TFLite tools"
run: |
gcloud storage cp "${TFLITE_ANDROID_BUILD_DIR_GCS_ARTIFACT}" "${TFLITE_ANDROID_BUILD_DIR_ARCHIVE}"
tar -xvf "${TFLITE_ANDROID_BUILD_DIR_ARCHIVE}"
- name: "Benchmarking TFLite on Android"
id: benchmark-tflite
env:
TFLITE_RESULTS_FILE: tflite-${{ env.TARGET_DEVICE }}.json
RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }}
run: |
RESULTS_PATH="${RESULTS_DIR}/${TFLITE_RESULTS_FILE}"
TFLITE_BENCHMARK_BINARY="${TFLITE_ANDROID_BUILD_DIR}/benchmark_model" \
./experimental/tflite/benchmark_tflite_android.sh \
"${TARGET_DEVICE}" \
"${RESULTS_PATH}"
gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/"
echo "tflite-results-file=${TFLITE_RESULTS_FILE}" >> "${GITHUB_OUTPUT}"
echo "tflite-results-gcs-path=${RESULTS_GCS_DIR}/${TFLITE_RESULTS_FILE}" >> "${GITHUB_OUTPUT}"
- name: "Generating report"
env:
IREE_RESULTS_FILE: ${{ steps.benchmark-iree.outputs.iree-results-file }}
IREE_RESULTS_GCS_PATH: ${{ steps.benchmark-iree.outputs.iree-results-gcs-path }}
TFLITE_RESULTS_FILE: ${{ steps.benchmark-tflite.outputs.tflite-results-file }}
TFLITE_RESULTS_GCS_PATH: ${{ steps.benchmark-tflite.outputs.tflite-results-gcs-path }}
SUMMARY_HTML: mobile_summary.html
SUMMARY_HTML_GCS_PATH: "gs://tflite-benchmark-artifacts/mobile"
BENCHMARK_GCS_DIR: ${{ needs.setup.outputs.benchmark-gcs-dir }}
RUN_TIMESTAMP: ${{ needs.setup.outputs.run-timestamp }}
run: |
gcloud storage cp "${IREE_RESULTS_GCS_PATH}" "${IREE_RESULTS_FILE}"
gcloud storage cp "${TFLITE_RESULTS_GCS_PATH}" "${TFLITE_RESULTS_FILE}"
./experimental/utils/generate_iree_vs_tflite_report.sh \
"${IREE_RESULTS_FILE}" \
"${TFLITE_RESULTS_FILE}" \
"${SUMMARY_HTML}"
gcloud storage cp "${SUMMARY_HTML}" "${BENCHMARK_GCS_DIR}/${SUMMARY_HTML}"
gcloud storage cp "${IREE_RESULTS_FILE}" "${SUMMARY_HTML_GCS_PATH}/latest/iree.json"
gcloud storage cp "${TFLITE_RESULTS_FILE}" "${SUMMARY_HTML_GCS_PATH}/latest/tflite.json"
gcloud storage cp "${SUMMARY_HTML}" "${SUMMARY_HTML_GCS_PATH}/latest/${SUMMARY_HTML}"
gcloud storage cp "${IREE_RESULTS_FILE}" "${SUMMARY_HTML_GCS_PATH}/${RUN_TIMESTAMP}/iree.json"
gcloud storage cp "${TFLITE_RESULTS_FILE}" "${SUMMARY_HTML_GCS_PATH}/${RUN_TIMESTAMP}/tflite.json"
gcloud storage cp "${SUMMARY_HTML}" "${SUMMARY_HTML_GCS_PATH}/${RUN_TIMESTAMP}/${SUMMARY_HTML}"