Reduce CPU benchmark run time (#116)

iree-org · Aug 7, 2023 · 217aca2 · 217aca2
1 parent 617631a
commit 217aca2
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 36 deletions.
diff --git a/comparative_benchmark/jax_xla/benchmark_all.sh b/comparative_benchmark/jax_xla/benchmark_all.sh
@@ -64,7 +64,7 @@ if [ "${TARGET_DEVICE}" = "a2-highgpu-1g" ]; then
   ITERATIONS=50
 elif [ "${TARGET_DEVICE}" = "c2-standard-16" ]; then
   BENCHMARK_NAMES=("${CPU_BENCHMARK_NAMES[@]}")
-  ITERATIONS=20
+  ITERATIONS=5
 else
   echo "Unsupported target device ${TARGET_DEVICE}."
   exit 1
@@ -82,16 +82,17 @@ for benchmark_name in "${BENCHMARK_NAMES[@]}"; do
     --verbose
 done
 
+# Disable for now as it takes too long to run.
 # If running on CPU, also benchmark XLA CPU-Next.
 # Use a lower number of iterations since CPU-Next is slow.
-if [ "${TARGET_DEVICE}" = "c2-standard-16" ]; then
-  for benchmark_name in "${BENCHMARK_NAMES[@]}"; do
-    "${TD}/run_benchmarks.py" \
-      --benchmark_name="${benchmark_name}" \
-      --target_device="${TARGET_DEVICE}" \
-      --output="${OUTPUT_PATH}" \
-      --iterations=3 \
-      --compiler="xla_cpu_next" \
-      --verbose
-  done
-fi
+# if [ "${TARGET_DEVICE}" = "c2-standard-16" ]; then
+#   for benchmark_name in "${BENCHMARK_NAMES[@]}"; do
+#     "${TD}/run_benchmarks.py" \
+#       --benchmark_name="${benchmark_name}" \
+#       --target_device="${TARGET_DEVICE}" \
+#       --output="${OUTPUT_PATH}" \
+#       --iterations=3 \
+#       --compiler="xla_cpu_next" \
+#       --verbose
+#   done
+# fi
diff --git a/comparative_benchmark/tf_xla/benchmark_all.sh b/comparative_benchmark/tf_xla/benchmark_all.sh
@@ -60,7 +60,7 @@ if [ "${TARGET_DEVICE}" = "a2-highgpu-1g" ]; then
   ITERATIONS=50
 elif [ "${TARGET_DEVICE}" = "c2-standard-16" ]; then
   BENCHMARK_NAMES=("${CPU_BENCHMARK_NAMES[@]}")
-  ITERATIONS=20
+  ITERATIONS=5
 else
   echo "Unsupported target device ${TARGET_DEVICE}."
   exit 1
@@ -78,16 +78,17 @@ for benchmark_name in "${BENCHMARK_NAMES[@]}"; do
     --verbose
 done
 
+# Disable for now as it takes too long to run.
 # If running on CPU, also benchmark XLA CPU-Next.
 # Use a lower number of iterations since CPU-Next is slow.
-if [ "${TARGET_DEVICE}" = "c2-standard-16" ]; then
-  for benchmark_name in "${BENCHMARK_NAMES[@]}"; do
-    "${TD}/run_benchmarks.py" \
-      --benchmark_name="${benchmark_name}" \
-      --target_device="${TARGET_DEVICE}" \
-      --output="${OUTPUT_PATH}" \
-      --iterations=3 \
-      --compiler="xla_cpu_next" \
-      --verbose
-  done
-fi
+# if [ "${TARGET_DEVICE}" = "c2-standard-16" ]; then
+#   for benchmark_name in "${BENCHMARK_NAMES[@]}"; do
+#     "${TD}/run_benchmarks.py" \
+#       --benchmark_name="${benchmark_name}" \
+#       --target_device="${TARGET_DEVICE}" \
+#       --output="${OUTPUT_PATH}" \
+#       --iterations=3 \
+#       --compiler="xla_cpu_next" \
+#       --verbose
+#   done
+# fi
diff --git a/comparative_benchmark/xla_hlo/benchmark_all.sh b/comparative_benchmark/xla_hlo/benchmark_all.sh
@@ -95,17 +95,18 @@ for benchmark_name in "${BENCHMARK_NAMES[@]}"; do
     --verbose
 done
 
+# Disable for now as it takes too long to run.
 # If running on CPU, also benchmark XLA CPU-Next.
 # Use a lower number of iterations since CPU-Next is slow.
-if [ "${TARGET_DEVICE}" = "c2-standard-16" ]; then
-  for benchmark_name in "${BENCHMARK_NAMES[@]}"; do
-    "${TD}/run_benchmarks.py" \
-      --benchmark_name="${benchmark_name}" \
-      --target_device="${TARGET_DEVICE}" \
-      --hlo-tool="${XLA_TOOLS_DIR}/${HLO_TOOL}" \
-      --output="${OUTPUT_PATH}" \
-      --iterations=3 \
-      --compiler="xla_cpu_next" \
-      --verbose
-  done
-fi
+# if [ "${TARGET_DEVICE}" = "c2-standard-16" ]; then
+#   for benchmark_name in "${BENCHMARK_NAMES[@]}"; do
+#     "${TD}/run_benchmarks.py" \
+#       --benchmark_name="${benchmark_name}" \
+#       --target_device="${TARGET_DEVICE}" \
+#       --hlo-tool="${XLA_TOOLS_DIR}/${HLO_TOOL}" \
+#       --output="${OUTPUT_PATH}" \
+#       --iterations=3 \
+#       --compiler="xla_cpu_next" \
+#       --verbose
+#   done
+# fi