Clone https://github.com/mlcommons/algorithmic-efficiency.git

.github/workflows/userbenchmark-a100-release.yml

@@ -1,8 +1,9 @@
 name: Release TorchBench Userbenchmark on A100
 on:
   pull_request:
-    paths:
-      - userbenchmark/release-test/*
+  push:
+    branches:
+      - main
 
 jobs:
   run-userbenchmark:
@@ -42,6 +43,12 @@ jobs:
           # remove old results
           if [ -d benchmark-output ]; then rm -Rf benchmark-output; fi
           pushd benchmark
+
+          # Install necessary packages  <-- Added these lines
+          pip3 install -e '.[jax_cpu]'
+          pip3 install -e '.[pytorch_gpu]' -f 'https://download.pytorch.org/whl/cu121'
+          pip3 install -e '.[full]'
+
           release_version=$(cat userbenchmark/release-test/version.txt)
           if [ -d .userbenchmark ]; then rm -Rf .userbenchmark; fi
           python run_benchmark.py release-test -c ${release_version}

install.py

@@ -11,6 +11,19 @@
 REPO_ROOT = Path(__file__).parent
 
 
+import argparse
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+from userbenchmark import list_userbenchmarks
+from utils import generate_pkg_constraints, get_pkg_versions, TORCH_DEPS
+from utils.python_utils import pip_install_requirements
+
+REPO_ROOT = Path(__file__).parent
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(allow_abbrev=False)
     parser.add_argument(

userbenchmark/release-test/result_analyzer.py

@@ -30,7 +30,9 @@ def dump_result_csv(work_dir, result):
     DELIMITER = ";"
     # generate header
     run_keys = sorted(result.keys())
+    print("run_keys:", run_keys)
     workloads = sorted(result[run_keys[0]])
+    print("workloads:", workloads)
     metrics = sorted(result[run_keys[0]][workloads[0]])
     for run_key in run_keys:
         csv_object[0].append(f"{run_key}")

userbenchmark/release-test/run.py

@@ -37,6 +37,38 @@
 bash {RELEASE_TEST_ROOT}/run_release_test.sh '{CUDA_VERSION}' '{RESULT_DIR}'
 """
 
+def run_algorithmic_efficiency(work_dir, experiment_dir, experiment_name):
+    """Runs algorithmic efficiency benchmarks."""
+    repo_dir = work_dir.joinpath("algorithmic_efficiency")
+    print("repo_dir:", repo_dir)
+    repo_url = "https://github.com/mlcommons/algorithmic-efficiency.git"
+
+    if not os.path.exists(repo_dir):
+        try:
+            Repo.clone_from(repo_url, repo_dir)
+        except Exception as e:
+            print(f"Error cloning algorithmic-efficiency repo using Repo.clone_from: {e}")
+            return False
+
+    command = [
+        "python3",
+        f"{repo_dir}/submission_runner.py",
+        "--workload=mnist",
+        "--framework=pytorch",
+        f"--submission_path={repo_dir}/reference_algorithms/development_algorithms/mnist/mnist_pytorch/submission.py",
+        "--tuning_ruleset=external",
+        f"--tuning_search_space={repo_dir}/reference_algorithms/development_algorithms/mnist/tuning_search_space.json",
+        "--num_tuning_trials=3",  # Adjust as needed
+        f"--experiment_dir={experiment_dir}",
+        f"--experiment_name={experiment_name}",
+    ]
+    print("running command:", command)
+
+    ret = subprocess.call(command)
+    if ret != 0:
+        print(f"Error running algorithmic efficiency benchmark: {ret}")
+        return False
+    return True
 
 def get_timestamp():
     return datetime.fromtimestamp(time.time()).strftime("%Y%m%d%H%M%S")
@@ -77,6 +109,7 @@ def dump_test_scripts(run_scripts, work_dir):
         run_script_loc = work_dir.joinpath(run_key)
         run_script_loc.mkdir(exist_ok=True)
         with open(run_script_loc.joinpath("run.sh"), "w") as rs:
+            print("writing run_script:", run_script)
             rs.write(run_script)
 
 
@@ -149,7 +182,13 @@ def run(args: List[str]):
     work_dir = get_work_dir(get_output_dir(BM_NAME))
     run_scripts = prepare_release_tests(args=args, work_dir=work_dir)
     if not args.dry_run:
+        # Run algorithmic efficiency benchmarks
+        print("Running run_algorithmic_efficiency starts...")
+        experiment_dir = os.path.join(work_dir, "algorithmic_efficiency_results")  # Create a subdirectory
+        experiment_name = f"algorithmic_efficiency_{get_timestamp()}"
+        run_algorithmic_efficiency(work_dir, experiment_dir, experiment_name)
         run_benchmark(run_scripts, work_dir)
+        print("analyze work_dir:", work_dir)
         metrics = analyze(work_dir)
         dump_result_to_json(metrics)
         cleanup_release_tests(work_dir)