diff --git a/.github/workflows/userbenchmark-a100-release.yml b/.github/workflows/userbenchmark-a100-release.yml index 6523af969..ea77e03fb 100644 --- a/.github/workflows/userbenchmark-a100-release.yml +++ b/.github/workflows/userbenchmark-a100-release.yml @@ -42,6 +42,12 @@ jobs: # remove old results if [ -d benchmark-output ]; then rm -Rf benchmark-output; fi pushd benchmark + + # Install necessary packages <-- Added these lines + pip3 install -e '.[jax_cpu]' + pip3 install -e '.[pytorch_gpu]' -f 'https://download.pytorch.org/whl/cu121' + pip3 install -e '.[full]' + release_version=$(cat userbenchmark/release-test/version.txt) if [ -d .userbenchmark ]; then rm -Rf .userbenchmark; fi python run_benchmark.py release-test -c ${release_version} diff --git a/install.py b/install.py index cc8e52a22..2b88a0c5c 100644 --- a/install.py +++ b/install.py @@ -11,6 +11,19 @@ REPO_ROOT = Path(__file__).parent +import argparse +import os +import subprocess +import sys +from pathlib import Path + +from userbenchmark import list_userbenchmarks +from utils import generate_pkg_constraints, get_pkg_versions, TORCH_DEPS +from utils.python_utils import pip_install_requirements + +REPO_ROOT = Path(__file__).parent + + if __name__ == "__main__": parser = argparse.ArgumentParser(allow_abbrev=False) parser.add_argument( diff --git a/userbenchmark/release-test/result_analyzer.py b/userbenchmark/release-test/result_analyzer.py index 7f79330ab..31a2da830 100644 --- a/userbenchmark/release-test/result_analyzer.py +++ b/userbenchmark/release-test/result_analyzer.py @@ -30,7 +30,9 @@ def dump_result_csv(work_dir, result): DELIMITER = ";" # generate header run_keys = sorted(result.keys()) + print("run_keys:", run_keys) workloads = sorted(result[run_keys[0]]) + print("workloads:", workloads) metrics = sorted(result[run_keys[0]][workloads[0]]) for run_key in run_keys: csv_object[0].append(f"{run_key}") diff --git a/userbenchmark/release-test/run.py b/userbenchmark/release-test/run.py index 6f17f6583..4ffb0339f 100644 --- a/userbenchmark/release-test/run.py +++ b/userbenchmark/release-test/run.py @@ -37,6 +37,38 @@ bash {RELEASE_TEST_ROOT}/run_release_test.sh '{CUDA_VERSION}' '{RESULT_DIR}' """ +def run_algorithmic_efficiency(work_dir, experiment_dir, experiment_name): + """Runs algorithmic efficiency benchmarks.""" + repo_dir = work_dir.joinpath("algorithmic_efficiency") + print("repo_dir:", repo_dir) + repo_url = "https://github.com/mlcommons/algorithmic-efficiency.git" + + if not os.path.exists(repo_dir): + try: + Repo.clone_from(repo_url, repo_dir) + except Exception as e: + print(f"Error cloning algorithmic-efficiency repo using Repo.clone_from: {e}") + return False + + command = [ + "python3", + f"{repo_dir}/submission_runner.py", + "--workload=mnist", + "--framework=pytorch", + f"--submission_path={repo_dir}/reference_algorithms/development_algorithms/mnist/mnist_pytorch/submission.py", + "--tuning_ruleset=external", + f"--tuning_search_space={repo_dir}/reference_algorithms/development_algorithms/mnist/tuning_search_space.json", + "--num_tuning_trials=3", # Adjust as needed + f"--experiment_dir={experiment_dir}", + f"--experiment_name={experiment_name}", + ] + print("running command:", command) + + ret = subprocess.call(command) + if ret != 0: + print(f"Error running algorithmic efficiency benchmark: {ret}") + return False + return True def get_timestamp(): return datetime.fromtimestamp(time.time()).strftime("%Y%m%d%H%M%S") @@ -77,6 +109,7 @@ def dump_test_scripts(run_scripts, work_dir): run_script_loc = work_dir.joinpath(run_key) run_script_loc.mkdir(exist_ok=True) with open(run_script_loc.joinpath("run.sh"), "w") as rs: + print("writing run_script:", run_script) rs.write(run_script) @@ -149,7 +182,14 @@ def run(args: List[str]): work_dir = get_work_dir(get_output_dir(BM_NAME)) run_scripts = prepare_release_tests(args=args, work_dir=work_dir) if not args.dry_run: + # Run algorithmic efficiency benchmarks + print("Running run_algorithmic_efficiency starts...") + experiment_dir = os.path.join(work_dir, "algorithmic_efficiency_results") # Create a subdirectory + experiment_name = f"algorithmic_efficiency_{get_timestamp()}" + print("start run_algorithmic_efficiency") + run_algorithmic_efficiency(work_dir, experiment_dir, experiment_name) run_benchmark(run_scripts, work_dir) + print("analyze work_dir:", work_dir) metrics = analyze(work_dir) dump_result_to_json(metrics) cleanup_release_tests(work_dir) diff --git a/userbenchmark/release-test/setup_env.sh b/userbenchmark/release-test/setup_env.sh index 4f47bb497..1eb5a88d0 100644 --- a/userbenchmark/release-test/setup_env.sh +++ b/userbenchmark/release-test/setup_env.sh @@ -39,8 +39,10 @@ conda install -y -c pytorch ${MAGMA_VERSION} # install pip version of pytorch and torchvision if [[ ${PYTORCH_CHANNEL} == "pytorch-test" ]]; then pip3 install torch==${PYTORCH_VERSION} torchvision --index-url https://download.pytorch.org/whl/test/cu${CUDA_VERSION//./} + pip3 install jax else pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cu${CUDA_VERSION//./} + pip3 install jax fi python -c 'import torch; print(torch.__version__); print(torch.version.git_version)'