From 399aeec29de038a320d56cd27784c28df6a2b867 Mon Sep 17 00:00:00 2001 From: Jack Luar Date: Thu, 24 Apr 2025 19:02:44 +0000 Subject: [PATCH 1/3] add copy and teardown for metadata.json for algoeval test - because clean_all was used, the original metadata.json was nuked. Signed-off-by: Jack Luar --- tools/AutoTuner/test/smoke_test_algo_eval.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/AutoTuner/test/smoke_test_algo_eval.py b/tools/AutoTuner/test/smoke_test_algo_eval.py index 01fdc95403..9c4e62dfe4 100644 --- a/tools/AutoTuner/test/smoke_test_algo_eval.py +++ b/tools/AutoTuner/test/smoke_test_algo_eval.py @@ -35,6 +35,7 @@ import unittest import subprocess import os +import shutil from .autotuner_test_utils import AutoTunerTestUtils, accepted_rc cur_dir = os.path.dirname(os.path.abspath(__file__)) @@ -66,6 +67,9 @@ def setUp(self): f" --reference {self.reference}" for a, e in self.matrix ] + # Make a file copy of the original metadata.json + self.metadata = os.path.join(cur_dir, f"{design_path}/metadata.json") + shutil.copyfile(self.metadata, self.metadata + ".orig") def make_base(self): commands = [ @@ -88,6 +92,10 @@ def test_algo_eval(self): successful = out.returncode in accepted_rc self.assertTrue(successful) + # On successful run, restore the metadata.json file + shutil.copyfile(self.metadata + ".orig", self.metadata) + os.remove(self.metadata + ".orig") + class asap7AlgoEvalSmokeTest(BaseAlgoEvalSmokeTest): platform = "asap7" From 7746a425c2ea6b749539963e9a2bf3747a40c487 Mon Sep 17 00:00:00 2001 From: Jack Luar Date: Thu, 24 Apr 2025 19:05:20 +0000 Subject: [PATCH 2/3] fix: report_path Signed-off-by: Jack Luar --- tools/AutoTuner/test/smoke_test_algo_eval.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/AutoTuner/test/smoke_test_algo_eval.py b/tools/AutoTuner/test/smoke_test_algo_eval.py index 9c4e62dfe4..fd1aa60f08 100644 --- a/tools/AutoTuner/test/smoke_test_algo_eval.py +++ b/tools/AutoTuner/test/smoke_test_algo_eval.py @@ -48,6 +48,7 @@ class BaseAlgoEvalSmokeTest(unittest.TestCase): def setUp(self): design_path = f"../../../flow/designs/{self.platform}/{self.design}" + report_path = f"../../../flow/reports/{self.platform}/{self.design}" self.config = os.path.join(cur_dir, f"{design_path}/autotuner.json") self.experiment = f"smoke-test-algo-eval-{self.platform}" self.reference = os.path.join(cur_dir, f"{design_path}/metadata-base-at.json") @@ -68,7 +69,7 @@ def setUp(self): for a, e in self.matrix ] # Make a file copy of the original metadata.json - self.metadata = os.path.join(cur_dir, f"{design_path}/metadata.json") + self.metadata = os.path.join(cur_dir, f"{report_path}/metadata.json") shutil.copyfile(self.metadata, self.metadata + ".orig") def make_base(self): From 399356c6f2983a6f422bc62d8b2d3807d8348a03 Mon Sep 17 00:00:00 2001 From: Jack Luar Date: Thu, 1 May 2025 08:12:38 +0000 Subject: [PATCH 3/3] create test-specific artifacts for smoke-test-algo-eval Signed-off-by: Jack Luar --- tools/AutoTuner/test/smoke_test_algo_eval.py | 26 ++++++++------------ 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/tools/AutoTuner/test/smoke_test_algo_eval.py b/tools/AutoTuner/test/smoke_test_algo_eval.py index fd1aa60f08..ad3d53284f 100644 --- a/tools/AutoTuner/test/smoke_test_algo_eval.py +++ b/tools/AutoTuner/test/smoke_test_algo_eval.py @@ -35,7 +35,6 @@ import unittest import subprocess import os -import shutil from .autotuner_test_utils import AutoTunerTestUtils, accepted_rc cur_dir = os.path.dirname(os.path.abspath(__file__)) @@ -45,13 +44,15 @@ class BaseAlgoEvalSmokeTest(unittest.TestCase): platform = "" design = "" + flow_variant = "smoke-test-algo-eval" def setUp(self): design_path = f"../../../flow/designs/{self.platform}/{self.design}" - report_path = f"../../../flow/reports/{self.platform}/{self.design}" self.config = os.path.join(cur_dir, f"{design_path}/autotuner.json") - self.experiment = f"smoke-test-algo-eval-{self.platform}" - self.reference = os.path.join(cur_dir, f"{design_path}/metadata-base-at.json") + self.experiment = f"{self.flow_variant}-{self.platform}" + self.reference = os.path.join( + cur_dir, f"{design_path}/metadata-{self.flow_variant}-at.json" + ) # note for ppa-improv, you need to also add in reference file (--reference) _algo = ["hyperopt", "ax", "optuna", "pbt", "random"] _eval = ["default", "ppa-improv"] @@ -68,17 +69,14 @@ def setUp(self): f" --reference {self.reference}" for a, e in self.matrix ] - # Make a file copy of the original metadata.json - self.metadata = os.path.join(cur_dir, f"{report_path}/metadata.json") - shutil.copyfile(self.metadata, self.metadata + ".orig") def make_base(self): - commands = [ - f"make -C {orfs_dir} DESIGN_CONFIG=./designs/{self.platform}/{self.design}/config.mk clean_all", - f"make -C {orfs_dir} DESIGN_CONFIG=./designs/{self.platform}/{self.design}/config.mk EQUIVALENCE_CHECK=0", - f"make -C {orfs_dir} DESIGN_CONFIG=./designs/{self.platform}/{self.design}/config.mk update_metadata_autotuner", + base_commands = [ + f"FLOW_VARIANT={self.flow_variant} make -C {orfs_dir} DESIGN_CONFIG=./designs/{self.platform}/{self.design}/config.mk clean_all", + f"FLOW_VARIANT={self.flow_variant} make -C {orfs_dir} DESIGN_CONFIG=./designs/{self.platform}/{self.design}/config.mk EQUIVALENCE_CHECK=0", + f"FLOW_VARIANT={self.flow_variant} make -C {orfs_dir} DESIGN_CONFIG=./designs/{self.platform}/{self.design}/config.mk update_metadata_autotuner", ] - for command in commands: + for command in base_commands: out = subprocess.run(command, shell=True) self.assertTrue(out.returncode in accepted_rc) @@ -93,10 +91,6 @@ def test_algo_eval(self): successful = out.returncode in accepted_rc self.assertTrue(successful) - # On successful run, restore the metadata.json file - shutil.copyfile(self.metadata + ".orig", self.metadata) - os.remove(self.metadata + ".orig") - class asap7AlgoEvalSmokeTest(BaseAlgoEvalSmokeTest): platform = "asap7"