From 800300d6c64e507b82d8e925b0a9d1f3894b2f52 Mon Sep 17 00:00:00 2001 From: Jack Luar Date: Sat, 7 Sep 2024 11:55:18 +0000 Subject: [PATCH 1/5] initial scaffold Signed-off-by: Jack Luar --- flow/designs/asap7/aes/autotuner.json | 8 - flow/designs/asap7/gcd/autotuner.json | 4 +- flow/designs/asap7/ibex/autotuner.json | 8 - flow/designs/ihp-sg13g2/gcd/autotuner.json | 8 - flow/designs/ihp-sg13g2/ibex/autotuner.json | 8 - flow/designs/sky130hd/aes/autotuner.json | 8 - flow/designs/sky130hd/gcd/autotuner.json | 8 - flow/designs/sky130hd/ibex/autotuner.json | 8 - flow/test/test_helper.sh | 15 +- tools/AutoTuner/.gitignore | 3 + tools/AutoTuner/test/regression_tune_base.py | 170 +++++++++++++++++++ 11 files changed, 182 insertions(+), 66 deletions(-) create mode 100644 tools/AutoTuner/test/regression_tune_base.py diff --git a/flow/designs/asap7/aes/autotuner.json b/flow/designs/asap7/aes/autotuner.json index 8430818c92..d5bfbf1867 100644 --- a/flow/designs/asap7/aes/autotuner.json +++ b/flow/designs/asap7/aes/autotuner.json @@ -64,14 +64,6 @@ ], "step": 0 }, - "_PINS_DISTANCE": { - "type": "int", - "minmax": [ - 1, - 1 - ], - "step": 1 - }, "CTS_CLUSTER_SIZE": { "type": "int", "minmax": [ diff --git a/flow/designs/asap7/gcd/autotuner.json b/flow/designs/asap7/gcd/autotuner.json index 5070c35bb0..11523e8e87 100644 --- a/flow/designs/asap7/gcd/autotuner.json +++ b/flow/designs/asap7/gcd/autotuner.json @@ -3,8 +3,8 @@ "_SDC_CLK_PERIOD": { "type": "float", "minmax": [ - 300, - 1000 + 50, + 500 ], "step": 0 }, diff --git a/flow/designs/asap7/ibex/autotuner.json b/flow/designs/asap7/ibex/autotuner.json index 08d679909b..9a23570b60 100644 --- a/flow/designs/asap7/ibex/autotuner.json +++ b/flow/designs/asap7/ibex/autotuner.json @@ -64,14 +64,6 @@ ], "step": 0 }, - "_PINS_DISTANCE": { - "type": "int", - "minmax": [ - 1, - 1 - ], - "step": 1 - }, "CTS_CLUSTER_SIZE": { "type": "int", "minmax": [ diff --git a/flow/designs/ihp-sg13g2/gcd/autotuner.json b/flow/designs/ihp-sg13g2/gcd/autotuner.json index d349a2322f..813c144bc5 100644 --- a/flow/designs/ihp-sg13g2/gcd/autotuner.json +++ b/flow/designs/ihp-sg13g2/gcd/autotuner.json @@ -64,14 +64,6 @@ ], "step": 0 }, - "_PINS_DISTANCE": { - "type": "int", - "minmax": [ - 1, - 1 - ], - "step": 1 - }, "CTS_CLUSTER_SIZE": { "type": "int", "minmax": [ diff --git a/flow/designs/ihp-sg13g2/ibex/autotuner.json b/flow/designs/ihp-sg13g2/ibex/autotuner.json index 1ed750a327..7afc0ffa45 100644 --- a/flow/designs/ihp-sg13g2/ibex/autotuner.json +++ b/flow/designs/ihp-sg13g2/ibex/autotuner.json @@ -64,14 +64,6 @@ ], "step": 0 }, - "_PINS_DISTANCE": { - "type": "int", - "minmax": [ - 1, - 4 - ], - "step": 1 - }, "CTS_CLUSTER_SIZE": { "type": "int", "minmax": [ diff --git a/flow/designs/sky130hd/aes/autotuner.json b/flow/designs/sky130hd/aes/autotuner.json index 4646b8f11b..fe9dfc74c7 100644 --- a/flow/designs/sky130hd/aes/autotuner.json +++ b/flow/designs/sky130hd/aes/autotuner.json @@ -64,14 +64,6 @@ ], "step": 0 }, - "_PINS_DISTANCE": { - "type": "int", - "minmax": [ - 1, - 1 - ], - "step": 1 - }, "CTS_CLUSTER_SIZE": { "type": "int", "minmax": [ diff --git a/flow/designs/sky130hd/gcd/autotuner.json b/flow/designs/sky130hd/gcd/autotuner.json index 6d12dd634d..7cea6da706 100644 --- a/flow/designs/sky130hd/gcd/autotuner.json +++ b/flow/designs/sky130hd/gcd/autotuner.json @@ -65,14 +65,6 @@ ], "step": 0 }, - "_PINS_DISTANCE": { - "type": "int", - "minmax": [ - 1, - 1 - ], - "step": 1 - }, "CTS_CLUSTER_SIZE": { "type": "int", "minmax": [ diff --git a/flow/designs/sky130hd/ibex/autotuner.json b/flow/designs/sky130hd/ibex/autotuner.json index 0c2c84e447..ea76f28ac9 100644 --- a/flow/designs/sky130hd/ibex/autotuner.json +++ b/flow/designs/sky130hd/ibex/autotuner.json @@ -64,14 +64,6 @@ ], "step": 0 }, - "_PINS_DISTANCE": { - "type": "int", - "minmax": [ - 1, - 1 - ], - "step": 1 - }, "CTS_CLUSTER_SIZE": { "type": "int", "minmax": [ diff --git a/flow/test/test_helper.sh b/flow/test/test_helper.sh index 0d0ebecd10..607b85952b 100755 --- a/flow/test/test_helper.sh +++ b/flow/test/test_helper.sh @@ -84,14 +84,13 @@ case $DESIGN_NAME in RUN_AUTOTUNER=1 ;; esac -case $PLATFORM in - "asap7" | "sky130hd" | "ihp-sg13g2" ) - # Keep RUN_AUTOTUNER enabled only for these platforms - ;; - *) - RUN_AUTOTUNER=0 - ;; -esac +if [ $RUN_AUTOTUNER -eq 1 ]; then + case $PLATFORM in + "gf180" | "nangate45" | "sky130hd_fakestack" | "sky130hs") + RUN_AUTOTUNER=0 + ;; + esac +fi if [ $RUN_AUTOTUNER -eq 1 ]; then # change directory to the root of the repo diff --git a/tools/AutoTuner/.gitignore b/tools/AutoTuner/.gitignore index 36ec367c7b..d073ce530a 100644 --- a/tools/AutoTuner/.gitignore +++ b/tools/AutoTuner/.gitignore @@ -9,3 +9,6 @@ __pycache__/ # Autotuner env autotuner_env +*.log +.env +*.zip diff --git a/tools/AutoTuner/test/regression_tune_base.py b/tools/AutoTuner/test/regression_tune_base.py new file mode 100644 index 0000000000..0abc21aa3a --- /dev/null +++ b/tools/AutoTuner/test/regression_tune_base.py @@ -0,0 +1,170 @@ +import unittest +import subprocess +import os +import json +import pandas as pd +import glob + +cur_dir = os.path.dirname(os.path.abspath(__file__)) +src_dir = os.path.join(cur_dir, "../src/autotuner") +os.chdir(src_dir) + + +def get_latest_date(path): + return max(glob.glob(f"{path}/*"), key=os.path.getmtime) + + +def combine_csv(path): + df = pd.concat( + [pd.read_csv(fname) for fname in glob.glob(f"{path}/*/progress.csv")] + ) + df.to_csv(f"{path}/progress_overall.csv", index=False) + + +def load_df(filename): + df = pd.read_csv(filename) + cols_to_remove = [ + "done", + "training_iteration", + "trial_id", + "date", + "timestamp", + "pid", + "hostname", + "node_ip", + "time_since_restore", + "time_total_s", + "iterations_since_restore", + ] + df = df[df["minimum"] != 9e99] + df = df.drop(columns=cols_to_remove) + return df + + +def get_latest_data(path): + latest_path = get_latest_date(path) + print("Using folder:", latest_path) + combine_csv(latest_path) + return load_df(f"{latest_path}/progress_overall.csv") + + +class BaseTuneRegressionBaseTest(unittest.TestCase): + platform = "" + design = "" + qor = 0 + THRESHOLD = 0.25 # qor median is within +- 25% of the expected value + + def setUp(self): + self.config = os.path.join( + cur_dir, + f"../../../flow/designs/{self.platform}/{self.design}/autotuner.json", + ) + self.experiment = f"test-regression" + self.command = ( + "python3 distributed.py" + f" --design {self.design}" + f" --platform {self.platform}" + f" --experiment {self.experiment}" + f" --config {self.config}" + f" tune --samples 10" + ) + + +class ASAP7TuneRegressionGCDTest(BaseTuneRegressionBaseTest): + platform = "asap7" + design = "gcd" + qor = 37668.290 + + def test_tune(self): + out = subprocess.run(self.command, shell=True, check=True) + successful = out.returncode == 0 + self.assertTrue(successful) + + # check if final mean QoR is within confidence interval of expected value + df = get_latest_data(f"../../../../flow/logs/{self.platform}/{self.design}") + median_qor = df["minimum"].median() + self.assertLess(abs(median_qor - self.qor) / self.qor, self.THRESHOLD) + + +class SKY130HDTuneRegressionGCDTest(BaseTuneRegressionBaseTest): + platform = "sky130hd" + design = "gcd" + qor = 327.023 + + def test_tune(self): + out = subprocess.run(self.command, shell=True, check=True) + successful = out.returncode == 0 + self.assertTrue(successful) + + # check if final mean QoR is within confidence interval of expected value + df = get_latest_data(f"../../../../flow/logs/{self.platform}/{self.design}") + median_qor = df["minimum"].median() + self.assertLess(abs(median_qor - self.qor) / self.qor, self.THRESHOLD) + + +class IHPSG13G2TuneRegressionGCDTest(BaseTuneRegressionBaseTest): + platform = "ihp-sg13g2" + design = "gcd" + qor = 251.102 + + def test_tune(self): + out = subprocess.run(self.command, shell=True, check=True) + successful = out.returncode == 0 + self.assertTrue(successful) + + # check if final mean QoR is within confidence interval of expected value + df = get_latest_data(f"../../../../flow/logs/{self.platform}/{self.design}") + median_qor = df["minimum"].median() + self.assertLess(abs(median_qor - self.qor) / self.qor, self.THRESHOLD) + + +class ASAP7TuneRegressionAESTest(BaseTuneRegressionBaseTest): + platform = "asap7" + design = "aes" + qor = 70934.140 + + def test_tune(self): + out = subprocess.run(self.command, shell=True, check=True) + successful = out.returncode == 0 + self.assertTrue(successful) + + # check if final mean QoR is within confidence interval of expected value + df = get_latest_data(f"../../../../flow/logs/{self.platform}/{self.design}") + mean_qor = df["minimum"].mean() + self.assertLess(abs(mean_qor - self.qor) / self.qor, (1 - self.THRESHOLD)) + + +class SKY130HDTuneRegressionAESTest(BaseTuneRegressionBaseTest): + platform = "sky130hd" + design = "aes" + qor = 546.433 + + def test_tune(self): + out = subprocess.run(self.command, shell=True, check=True) + successful = out.returncode == 0 + self.assertTrue(successful) + + # check if final mean QoR is within confidence interval of expected value + df = get_latest_data(f"../../../../flow/logs/{self.platform}/{self.design}") + mean_qor = df["minimum"].mean() + self.assertLess(abs(mean_qor - self.qor) / self.qor, (1 - self.THRESHOLD)) + + +class IHPSG13G2TuneRegressionAESTest(BaseTuneRegressionBaseTest): + platform = "ihp-sg13g2" + design = "aes" + qor = 544.835 + + def test_tune(self): + out = subprocess.run(self.command, shell=True, check=True) + successful = out.returncode == 0 + self.assertTrue(successful) + + # check if final mean QoR is within confidence interval of expected value + df = get_latest_data(f"../../../../flow/logs/{self.platform}/{self.design}") + mean_qor = df["minimum"].mean() + self.assertLess(abs(mean_qor - self.qor) / self.qor, (1 - self.THRESHOLD)) + + +if __name__ == "__main__": + unittest.main() From 803b895ec2a1b917922500d697e773cdba9f10dc Mon Sep 17 00:00:00 2001 From: Jack Luar Date: Sat, 7 Sep 2024 15:38:52 +0000 Subject: [PATCH 2/5] fix test_helper Signed-off-by: Jack Luar --- flow/test/test_helper.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/flow/test/test_helper.sh b/flow/test/test_helper.sh index 607b85952b..17df9585b0 100755 --- a/flow/test/test_helper.sh +++ b/flow/test/test_helper.sh @@ -80,7 +80,7 @@ fi # Run Autotuner CI specifically for gcd on selected platforms. RUN_AUTOTUNER=0 case $DESIGN_NAME in - "gcd") + "gcd" "aes" "ibex") RUN_AUTOTUNER=1 ;; esac @@ -103,12 +103,10 @@ if [ $RUN_AUTOTUNER -eq 1 ]; then PLATFORM=${PLATFORM//-/} # convert to uppercase PLATFORM=${PLATFORM^^} + DESIGN_NAME=${DESIGN_NAME^^} - echo "Running Autotuner smoke tune test" - python3 -m unittest tools.AutoTuner.test.smoke_test_tune.${PLATFORM}TuneSmokeTest.test_tune - - echo "Running Autotuner smoke sweep test" - python3 -m unittest tools.AutoTuner.test.smoke_test_sweep.${PLATFORM}SweepSmokeTest.test_sweep + echo "Running Autotuner Regression Test" + python3 -m unittest tools.AutoTuner.test.regression_tune_base.${PLATFORM}TuneRegression${DESIGN_NAME}Test.test_tune fi exit $ret From 5bc7c7779baad7305452dfaa8fe1770c187c6642 Mon Sep 17 00:00:00 2001 From: Jack Luar Date: Mon, 9 Sep 2024 01:56:33 +0000 Subject: [PATCH 3/5] typo in test_helper Signed-off-by: Jack Luar --- flow/test/test_helper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flow/test/test_helper.sh b/flow/test/test_helper.sh index 17df9585b0..248e8fb1cf 100755 --- a/flow/test/test_helper.sh +++ b/flow/test/test_helper.sh @@ -80,7 +80,7 @@ fi # Run Autotuner CI specifically for gcd on selected platforms. RUN_AUTOTUNER=0 case $DESIGN_NAME in - "gcd" "aes" "ibex") + "gcd" | "aes" | "ibex") RUN_AUTOTUNER=1 ;; esac From 0c127e12763734b263dbc6c8273215c87da2c7f5 Mon Sep 17 00:00:00 2001 From: Jack Luar Date: Mon, 9 Sep 2024 15:19:21 +0000 Subject: [PATCH 4/5] fix ibex missing tests Signed-off-by: Jack Luar --- tools/AutoTuner/test/regression_tune_base.py | 51 ++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tools/AutoTuner/test/regression_tune_base.py b/tools/AutoTuner/test/regression_tune_base.py index 0abc21aa3a..589b9c83f6 100644 --- a/tools/AutoTuner/test/regression_tune_base.py +++ b/tools/AutoTuner/test/regression_tune_base.py @@ -69,6 +69,9 @@ def setUp(self): f" tune --samples 10" ) + def test_tune(self): + raise NotImplementedError("Subclasses must implement this method.") + class ASAP7TuneRegressionGCDTest(BaseTuneRegressionBaseTest): platform = "asap7" @@ -166,5 +169,53 @@ def test_tune(self): self.assertLess(abs(mean_qor - self.qor) / self.qor, (1 - self.THRESHOLD)) +class ASAP7TuneRegressionIBEXTest(BaseTuneRegressionBaseTest): + platform = "asap7" + design = "ibex" + qor = 205678.545 + + def test_tune(self): + out = subprocess.run(self.command, shell=True, check=True) + successful = out.returncode == 0 + self.assertTrue(successful) + + # check if final mean QoR is within confidence interval of expected value + df = get_latest_data(f"../../../../flow/logs/{self.platform}/{self.design}") + mean_qor = df["minimum"].mean() + self.assertLess(abs(mean_qor - self.qor) / self.qor, (1 - self.THRESHOLD)) + + +class SKY130HDTuneRegressionIBEXTest(BaseTuneRegressionBaseTest): + platform = "sky130hd" + design = "ibex" + qor = 1425.996 + + def test_tune(self): + out = subprocess.run(self.command, shell=True, check=True) + successful = out.returncode == 0 + self.assertTrue(successful) + + # check if final mean QoR is within confidence interval of expected value + df = get_latest_data(f"../../../../flow/logs/{self.platform}/{self.design}") + mean_qor = df["minimum"].mean() + self.assertLess(abs(mean_qor - self.qor) / self.qor, (1 - self.THRESHOLD)) + + +class IHPSG13G2TuneRegressionIBEXTest(BaseTuneRegressionBaseTest): + platform = "ihp-sg13g2" + design = "ibex" + qor = 1422.309 + + def test_tune(self): + out = subprocess.run(self.command, shell=True, check=True) + successful = out.returncode == 0 + self.assertTrue(successful) + + # check if final mean QoR is within confidence interval of expected value + df = get_latest_data(f"../../../../flow/logs/{self.platform}/{self.design}") + mean_qor = df["minimum"].mean() + self.assertLess(abs(mean_qor - self.qor) / self.qor, (1 - self.THRESHOLD)) + + if __name__ == "__main__": unittest.main() From 3b540b6854d5dc42472fe3634a9dae101993844a Mon Sep 17 00:00:00 2001 From: Jack Luar Date: Sun, 29 Sep 2024 04:54:41 +0000 Subject: [PATCH 5/5] update latest regression Signed-off-by: Jack Luar --- tools/AutoTuner/test/regression_tune_base.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/AutoTuner/test/regression_tune_base.py b/tools/AutoTuner/test/regression_tune_base.py index 589b9c83f6..7264b0a6a3 100644 --- a/tools/AutoTuner/test/regression_tune_base.py +++ b/tools/AutoTuner/test/regression_tune_base.py @@ -76,7 +76,7 @@ def test_tune(self): class ASAP7TuneRegressionGCDTest(BaseTuneRegressionBaseTest): platform = "asap7" design = "gcd" - qor = 37668.290 + qor = 35847.475 def test_tune(self): out = subprocess.run(self.command, shell=True, check=True) @@ -92,7 +92,7 @@ def test_tune(self): class SKY130HDTuneRegressionGCDTest(BaseTuneRegressionBaseTest): platform = "sky130hd" design = "gcd" - qor = 327.023 + qor = 332.139 def test_tune(self): out = subprocess.run(self.command, shell=True, check=True) @@ -108,7 +108,7 @@ def test_tune(self): class IHPSG13G2TuneRegressionGCDTest(BaseTuneRegressionBaseTest): platform = "ihp-sg13g2" design = "gcd" - qor = 251.102 + qor = 398.452 def test_tune(self): out = subprocess.run(self.command, shell=True, check=True) @@ -124,7 +124,7 @@ def test_tune(self): class ASAP7TuneRegressionAESTest(BaseTuneRegressionBaseTest): platform = "asap7" design = "aes" - qor = 70934.140 + qor = 61455.680 def test_tune(self): out = subprocess.run(self.command, shell=True, check=True) @@ -140,7 +140,7 @@ def test_tune(self): class SKY130HDTuneRegressionAESTest(BaseTuneRegressionBaseTest): platform = "sky130hd" design = "aes" - qor = 546.433 + qor = 540.688 def test_tune(self): out = subprocess.run(self.command, shell=True, check=True) @@ -156,7 +156,7 @@ def test_tune(self): class IHPSG13G2TuneRegressionAESTest(BaseTuneRegressionBaseTest): platform = "ihp-sg13g2" design = "aes" - qor = 544.835 + qor = 544.728 def test_tune(self): out = subprocess.run(self.command, shell=True, check=True) @@ -172,7 +172,7 @@ def test_tune(self): class ASAP7TuneRegressionIBEXTest(BaseTuneRegressionBaseTest): platform = "asap7" design = "ibex" - qor = 205678.545 + qor = 192118.310 def test_tune(self): out = subprocess.run(self.command, shell=True, check=True) @@ -188,7 +188,7 @@ def test_tune(self): class SKY130HDTuneRegressionIBEXTest(BaseTuneRegressionBaseTest): platform = "sky130hd" design = "ibex" - qor = 1425.996 + qor = 1421.194 def test_tune(self): out = subprocess.run(self.command, shell=True, check=True) @@ -204,7 +204,7 @@ def test_tune(self): class IHPSG13G2TuneRegressionIBEXTest(BaseTuneRegressionBaseTest): platform = "ihp-sg13g2" design = "ibex" - qor = 1422.309 + qor = 1278.664 def test_tune(self): out = subprocess.run(self.command, shell=True, check=True)