From 22dcced2062ed0b29311b75a8908a01f32349294 Mon Sep 17 00:00:00 2001 From: Eryk Szpotanski Date: Fri, 11 Oct 2024 15:11:12 +0200 Subject: [PATCH 1/6] flow: scripts: detail_route: Report metrics Signed-off-by: Eryk Szpotanski --- flow/scripts/detail_route.tcl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flow/scripts/detail_route.tcl b/flow/scripts/detail_route.tcl index 330b93b49c..288c875052 100644 --- a/flow/scripts/detail_route.tcl +++ b/flow/scripts/detail_route.tcl @@ -68,6 +68,8 @@ if { [env_var_exists_and_non_empty POST_DETAIL_ROUTE_TCL] } { check_antennas -report_file $env(REPORTS_DIR)/drt_antennas.log +report_metrics 5 "detailed route" + if {![design_is_routed]} { error "Design has unrouted nets." } From c43c4a41d5972dec7e48c5ba2a011d13ef795747 Mon Sep 17 00:00:00 2001 From: Eryk Szpotanski Date: Mon, 21 Oct 2024 15:27:56 +0200 Subject: [PATCH 2/6] tools: Autotuner: Add dependencies for Vizier Signed-off-by: Eryk Szpotanski --- tools/AutoTuner/installer.sh | 13 ++++++++++++- tools/AutoTuner/pyproject.toml | 2 ++ tools/AutoTuner/requirements-ray.txt | 11 +++++++++++ tools/AutoTuner/requirements-vizier.txt | 3 +++ tools/AutoTuner/requirements.txt | 12 +----------- 5 files changed, 29 insertions(+), 12 deletions(-) create mode 100644 tools/AutoTuner/requirements-ray.txt create mode 100644 tools/AutoTuner/requirements-vizier.txt diff --git a/tools/AutoTuner/installer.sh b/tools/AutoTuner/installer.sh index 7d5f22f5fd..27cfbd77fc 100755 --- a/tools/AutoTuner/installer.sh +++ b/tools/AutoTuner/installer.sh @@ -3,9 +3,20 @@ # Get the directory where the script is located script_dir="$(dirname "${BASH_SOURCE[0]}")" +dependencies="" +if [[ "$#" -eq 0 ]]; then + echo "Installing dependencies for Ray Tune and Vizier" + dependencies="ray,vizier" +elif [[ "$#" -ne 1 ]] || ([[ "$1" != "ray" ]] && [[ "$1" != "vizier" ]]); then + echo "Please specify whether 'ray' or 'vizier' dependencies should be installed" >&2 + exit 1 +else + dependencies="$1" +fi + # Define the virtual environment name venv_name="autotuner_env" python3 -m venv "$script_dir/$venv_name" source "$script_dir/$venv_name/bin/activate" -pip3 install -e "$script_dir" +pip3 install -e "$script_dir[$dependencies]" deactivate diff --git a/tools/AutoTuner/pyproject.toml b/tools/AutoTuner/pyproject.toml index 3261ae831e..937803cbae 100644 --- a/tools/AutoTuner/pyproject.toml +++ b/tools/AutoTuner/pyproject.toml @@ -12,6 +12,8 @@ dynamic = ["dependencies", "optional-dependencies"] [tool.setuptools.dynamic] dependencies = { file = ["requirements.txt"] } +optional-dependencies.ray = { file = ["requirements-ray.txt"] } +optional-dependencies.vizier = { file = ["requirements-vizier.txt"] } optional-dependencies.dev = { file = ["requirements-dev.txt"] } [build-system] diff --git a/tools/AutoTuner/requirements-ray.txt b/tools/AutoTuner/requirements-ray.txt new file mode 100644 index 0000000000..5a09397ad0 --- /dev/null +++ b/tools/AutoTuner/requirements-ray.txt @@ -0,0 +1,11 @@ +ray[tune]==2.9.3 +ax-platform>=0.3.3,<=0.3.7 +hyperopt==0.2.7 +optuna==3.6.0 +pandas>=2.0,<=2.2.1 +bayesian-optimization==1.4.0 +colorama==0.4.6 +tensorboard>=2.14.0,<=2.16.2 +protobuf==3.20.3 +SQLAlchemy==1.4.17 +urllib3<=1.26.15 diff --git a/tools/AutoTuner/requirements-vizier.txt b/tools/AutoTuner/requirements-vizier.txt new file mode 100644 index 0000000000..0222aeddb4 --- /dev/null +++ b/tools/AutoTuner/requirements-vizier.txt @@ -0,0 +1,3 @@ +jax<=0.4.33 +google-vizier[jax] +tqdm diff --git a/tools/AutoTuner/requirements.txt b/tools/AutoTuner/requirements.txt index 5bf65305cc..e598feeda0 100644 --- a/tools/AutoTuner/requirements.txt +++ b/tools/AutoTuner/requirements.txt @@ -1,11 +1 @@ -ray[default,tune]==2.9.3 -ax-platform>=0.3.3,<=0.3.7 -hyperopt==0.2.7 -optuna==3.6.0 -pandas>=2.0,<=2.2.1 -bayesian-optimization==1.4.0 -colorama==0.4.6 -tensorboard>=2.14.0,<=2.16.2 -protobuf==3.20.3 -SQLAlchemy==1.4.17 -urllib3<=1.26.15 +ray[default]==2.9.3 From bc59e184eba63a3adc0c19d0685bbdccb6c7bb1f Mon Sep 17 00:00:00 2001 From: Eryk Szpotanski Date: Mon, 21 Oct 2024 15:37:07 +0200 Subject: [PATCH 3/6] tools: Autotuner: Implement Vizier support Signed-off-by: Eryk Szpotanski --- tools/AutoTuner/src/autotuner/distributed.py | 127 +---- tools/AutoTuner/src/autotuner/utils.py | 171 +++++++ tools/AutoTuner/src/autotuner/vizier.py | 497 +++++++++++++++++++ 3 files changed, 670 insertions(+), 125 deletions(-) create mode 100644 tools/AutoTuner/src/autotuner/vizier.py diff --git a/tools/AutoTuner/src/autotuner/distributed.py b/tools/AutoTuner/src/autotuner/distributed.py index fecf49ccfa..8e5295c9ff 100644 --- a/tools/AutoTuner/src/autotuner/distributed.py +++ b/tools/AutoTuner/src/autotuner/distributed.py @@ -32,7 +32,6 @@ from itertools import product from uuid import uuid4 as uuid from collections import namedtuple -from multiprocessing import cpu_count import numpy as np import torch @@ -51,6 +50,7 @@ from ax.service.ax_client import AxClient from autotuner.utils import ( + add_common_args, openroad, consumer, parse_config, @@ -69,8 +69,6 @@ ORFS_FLOW_DIR = os.path.abspath( os.path.join(os.path.dirname(__file__), "../../../../flow") ) -# URL to ORFS GitHub repository -ORFS_URL = "https://github.com/The-OpenROAD-Project/OpenROAD-flow-scripts" class AutoTunerBase(tune.Trainable): @@ -203,30 +201,9 @@ def parse_arguments(): tune_parser = subparsers.add_parser("tune") _ = subparsers.add_parser("sweep") - # DUT - parser.add_argument( - "--design", - type=str, - metavar="", - required=True, - help="Name of the design for Autotuning.", - ) - parser.add_argument( - "--platform", - type=str, - metavar="", - required=True, - help="Name of the platform for Autotuning.", - ) + add_common_args(parser) # Experiment Setup - parser.add_argument( - "--config", - type=str, - metavar="", - required=True, - help="Configuration file that sets which knobs to use for Autotuning.", - ) parser.add_argument( "--experiment", type=str, @@ -235,13 +212,6 @@ def parse_arguments(): help="Experiment name. This parameter is used to prefix the" " FLOW_VARIANT and to set the Ray log destination.", ) - parser.add_argument( - "--timeout", - type=float, - metavar="", - default=None, - help="Time limit (in hours) for each trial run. Default is no limit.", - ) tune_parser.add_argument( "--resume", action="store_true", @@ -249,60 +219,6 @@ def parse_arguments(): name identifier via `--experiment NAME` to be able to resume.", ) - # Setup - parser.add_argument( - "--git_clean", - action="store_true", - help="Clean binaries and build files." - " WARNING: may lose previous data." - " Use carefully.", - ) - parser.add_argument( - "--git_clone", - action="store_true", - help="Force new git clone." - " WARNING: may lose previous data." - " Use carefully.", - ) - parser.add_argument( - "--git_clone_args", - type=str, - metavar="", - default="", - help="Additional git clone arguments.", - ) - parser.add_argument( - "--git_latest", action="store_true", help="Use latest version of OpenROAD app." - ) - parser.add_argument( - "--git_or_branch", - type=str, - metavar="", - default="", - help="OpenROAD app branch to use.", - ) - parser.add_argument( - "--git_orfs_branch", - type=str, - metavar="", - default="master", - help="OpenROAD-flow-scripts branch to use.", - ) - parser.add_argument( - "--git_url", - type=str, - metavar="", - default=ORFS_URL, - help="OpenROAD-flow-scripts repo URL to use.", - ) - parser.add_argument( - "--build_args", - type=str, - metavar="", - default="", - help="Additional arguments given to ./build_openroad.sh.", - ) - # ML tune_parser.add_argument( "--algorithm", @@ -361,45 +277,6 @@ def parse_arguments(): help="Random seed. (0 means no seed.)", ) - # Workload - parser.add_argument( - "--jobs", - type=int, - metavar="", - default=int(np.floor(cpu_count() / 2)), - help="Max number of concurrent jobs.", - ) - parser.add_argument( - "--openroad_threads", - type=int, - metavar="", - default=16, - help="Max number of threads openroad can use.", - ) - parser.add_argument( - "--server", - type=str, - metavar="", - default=None, - help="The address of Ray server to connect.", - ) - parser.add_argument( - "--port", - type=int, - metavar="", - default=10001, - help="The port of Ray server to connect.", - ) - - parser.add_argument( - "-v", - "--verbose", - action="count", - default=0, - help="Verbosity level.\n\t0: only print Ray status\n\t1: also print" - " training stderr\n\t2: also print training stdout.", - ) - arguments = parser.parse_args() if arguments.mode == "tune": arguments.algorithm = arguments.algorithm.lower() diff --git a/tools/AutoTuner/src/autotuner/utils.py b/tools/AutoTuner/src/autotuner/utils.py index bc4bfdc091..d696f28afa 100644 --- a/tools/AutoTuner/src/autotuner/utils.py +++ b/tools/AutoTuner/src/autotuner/utils.py @@ -1,3 +1,4 @@ +import argparse import glob import json import os @@ -32,6 +33,8 @@ CONSTRAINTS_SDC = "constraint.sdc" # Name of the TCL script run before routing FASTROUTE_TCL = "fastroute.tcl" +# URL to ORFS GitHub repository +ORFS_URL = "https://github.com/The-OpenROAD-Project/OpenROAD-flow-scripts" DATE = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") @@ -296,6 +299,27 @@ def openroad( return metrics_file +STAGES = list( + enumerate( + [ + "synth", + "floorplan", + "floorplan_io", + "floorplan_tdms", + "floorplan_macro", + "floorplan_tap", + "floorplan_pdn", + "globalplace", + "detailedplace", + "cts", + "globalroute", + "detailedroute", + "finish", + ] + ) +) + + def read_metrics(file_name): """ Collects metrics to evaluate the user-defined objective function. @@ -312,6 +336,7 @@ def read_metrics(file_name): design_area = "ERR" die_area = "ERR" core_area = "ERR" + last_stage = -1 for stage_name, value in data.items(): if stage_name == "constraints" and len(value["clocks__details"]) > 0: clk_period = float(value["clocks__details"][0].split()[1]) @@ -333,6 +358,10 @@ def read_metrics(file_name): core_area = value["design__core__area"] if stage_name == "finish" and "design__die__area" in value: die_area = value["design__die__area"] + for i, stage_name in reversed(STAGES): + if stage_name in data and [d for d in data[stage_name].values() if d != "ERR"]: + last_stage = i + break ret = { "clk_period": clk_period, "worst_slack": worst_slack, @@ -467,6 +496,20 @@ def read_tune_pbt(name, this): if this["type"] == "float": return tune.uniform(min_, max_) + def read_vizier(this): + dict_ = {} + min_, max_ = this["minmax"] + dict_["value"] = (min_, max_) + if "scale_type" in this: + dict_["scale_type"] = this["scale_type"] + if min_ == max_: + dict_["type"] = "fixed" + elif this["type"] == "int": + dict_["type"] = "int" + elif this["type"] == "float": + dict_["type"] = "float" + return dict_ + # Check file exists and whether it is a valid JSON file. assert os.path.isfile(file_name), f"File {file_name} not found." try: @@ -513,6 +556,8 @@ def read_tune_pbt(name, this): config[key] = read_tune_pbt(key, value) elif mode == "tune": config[key] = read_tune(value) + elif mode == "vizier": + config[key] = read_vizier(value) if mode == "tune": config = apply_condition(config, data) return config, sdc_file, fr_file @@ -651,3 +696,129 @@ def consumer(queue): print(f"[INFO TUN-0007] Scheduling run for parameter {name}.") ray.get(openroad_distributed.remote(*next_item)) print(f"[INFO TUN-0008] Finished run for parameter {name}.") + + +def add_common_args(parser: argparse.ArgumentParser): + # DUT + parser.add_argument( + "--design", + type=str, + metavar="", + required=True, + help="Name of the design for Autotuning.", + ) + parser.add_argument( + "--platform", + type=str, + metavar="", + required=True, + help="Name of the platform for Autotuning.", + ) + # Experiment Setup + parser.add_argument( + "--config", + type=str, + metavar="", + required=True, + help="Configuration file that sets which knobs to use for Autotuning.", + ) + parser.add_argument( + "--timeout", + type=float, + metavar="", + default=None, + help="Time limit (in hours) for each trial run. Default is no limit.", + ) + # Workload + parser.add_argument( + "--openroad_threads", + type=int, + metavar="", + default=16, + help="Max number of threads openroad can use.", + ) + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Verbosity level.\n\t0: only print status\n\t1: also print" + " training stderr\n\t2: also print training stdout.", + ) + + # Setup + parser.add_argument( + "--git_clean", + action="store_true", + help="Clean binaries and build files." + " WARNING: may lose previous data." + " Use carefully.", + ) + parser.add_argument( + "--git_clone", + action="store_true", + help="Force new git clone." + " WARNING: may lose previous data." + " Use carefully.", + ) + parser.add_argument( + "--git_clone_args", + type=str, + metavar="", + default="", + help="Additional git clone arguments.", + ) + parser.add_argument( + "--git_latest", action="store_true", help="Use latest version of OpenROAD app." + ) + parser.add_argument( + "--git_or_branch", + type=str, + metavar="", + default="", + help="OpenROAD app branch to use.", + ) + parser.add_argument( + "--git_orfs_branch", + type=str, + metavar="", + default="master", + help="OpenROAD-flow-scripts branch to use.", + ) + parser.add_argument( + "--git_url", + type=str, + metavar="", + default=ORFS_URL, + help="OpenROAD-flow-scripts repo URL to use.", + ) + parser.add_argument( + "--build_args", + type=str, + metavar="", + default="", + help="Additional arguments given to ./build_openroad.sh.", + ) + + # Workload + parser.add_argument( + "--jobs", + type=int, + metavar="", + default=int(np.floor(cpu_count() / 2)), + help="Max number of concurrent jobs.", + ) + parser.add_argument( + "--server", + type=str, + metavar="", + default=None, + help="The address of Ray server to connect.", + ) + parser.add_argument( + "--port", + type=int, + metavar="", + default=10001, + help="The port of Ray server to connect.", + ) diff --git a/tools/AutoTuner/src/autotuner/vizier.py b/tools/AutoTuner/src/autotuner/vizier.py new file mode 100644 index 0000000000..225df742ed --- /dev/null +++ b/tools/AutoTuner/src/autotuner/vizier.py @@ -0,0 +1,497 @@ +import argparse +import json +import sys +import traceback +from pathlib import Path +from typing import Dict + +import ray +from tqdm import tqdm +from vizier import service +from vizier.service import clients, servers +from vizier.service import pyvizier as vz + +from autotuner.utils import ( + DATE, + add_common_args, + openroad_distributed, + read_config, + read_metrics, + prepare_ray_server, +) + +# Path to the ORFS base directory +ORFS = list(Path(__file__).absolute().parents)[4] +# Maps metrics to a goal (min or max) +METRIC_TO_GOAL = { + "worst_slack": vz.ObjectiveMetricGoal.MAXIMIZE, + "clk_period-worst_slack": vz.ObjectiveMetricGoal.MINIMIZE, + "total_power": vz.ObjectiveMetricGoal.MINIMIZE, + "core_util": vz.ObjectiveMetricGoal.MAXIMIZE, + "final_util": vz.ObjectiveMetricGoal.MAXIMIZE, + "design_area": vz.ObjectiveMetricGoal.MINIMIZE, + "core_area": vz.ObjectiveMetricGoal.MINIMIZE, + "die_area": vz.ObjectiveMetricGoal.MINIMIZE, + "last_successful_stage": vz.ObjectiveMetricGoal.MAXIMIZE, +} +# Maps goal to a worst value +GOAL_TO_VALUE = { + vz.ObjectiveMetricGoal.MINIMIZE: float("inf"), + vz.ObjectiveMetricGoal.MAXIMIZE: float("-inf"), +} +# Maps string to Vizier ScaleType +MAP_SCALE_TYPE = { + "linear": vz.ScaleType.LINEAR, + "log": vz.ScaleType.LOG, + "rlog": vz.ScaleType.REVERSE_LOG, +} + + +def evaluate(args: argparse.Namespace, metric_file: str) -> Dict[str, float]: + """ + Runs ORFS and calculates metrics. + + Parameters + ---------- + args : argparse.Namespace + Optimization arguments + metric_file : str + Path to the file with metrics + + Returns + ------- + Dict[str, float] + Dictionary with metrics + """ + try: + metrics = read_metrics(metric_file) + # Calculate difference of clock period and worst slack + if metrics["clk_period"] != 9999999 and metrics["worst_slack"] != "ERR": + metrics["clk_period-worst_slack"] = ( + metrics["clk_period"] - metrics["worst_slack"] + ) + else: + metrics["clk_period-worst_slack"] = "ERR" + + # Copy and normalize metrics + results = {} + for metric in args.use_metrics: + value = metrics[metric] + results[metric] = ( + float(value) + if value != "ERR" + else GOAL_TO_VALUE[METRIC_TO_GOAL[metric]] + ) + if results["last_successful_stage"] <= 6 and results["core_util"] < float( + "inf" + ): + # Invert core util, as for smaller values design should be easier to built + results["core_util"] *= -1 + return results + except Exception as ex: + print( + f"[ERROR TUN-0023] Exception during metrics processing {args.design}: {ex}", + file=sys.stderr, + ) + print("\n".join(traceback.format_tb(ex.__traceback__)), file=sys.stderr) + results = {} + for metric, goal in args.use_metrics: + results[metric] = GOAL_TO_VALUE[METRIC_TO_GOAL[metric]] + return results + + +@ray.remote +def parallel_evaluate( + args: argparse.Namespace, + suggestion: Dict, + i: int, + s: int, + install_path: Path, +) -> Dict: + """ + Wrapper for evaluate, run in thread pool. + + Parameters + ---------- + args : argparse.Namespace + Optimization arguments + suggestion : Dict + i : int + Number of iteration + s : int + Number of suggestion + install_path : Path + Path to the install directory with ORFS binaries + + Returns + ------- + Dict + Results of evaluation with additional data + """ + variant = f"variant-{i}-{s}" + metric_file, duration = ray.get( + openroad_distributed.remote( + args=args, + repo_dir=str(args.orfs), + config=suggestion, + path=f"logs/{args.platform}/{args.design}", + sdc_original=args.sdc_file, + fr_original=args.fr_file, + install_path=str(install_path), + variant=variant, + ) + ) + objective = evaluate(args, metric_file) + return { + "iterations": i, + "suggestion": s, + "params": suggestion, + "evaluation": objective, + "variant": variant, + "duration": duration, + } + + +def register_param( + args: argparse.Namespace, problem: vz.ProblemStatement, name: str, conf: Dict +): + """ + Registers parameters in Vizier problem statement. + + Parameters + ---------- + args : argparse.Namespace + Optimization arguments + problem : vz.ProblemStatement + Vizier problem statement + name : str + Name of the parameter + conf : Dict + Parameter config + """ + if conf["type"] == "fixed": + problem.search_space.root.add_discrete_param( + name, + feasible_values=[conf["value"][0]], + ) + else: + map_func = { + "float": problem.search_space.root.add_float_param, + "int": problem.search_space.root.add_int_param, + } + map_func[conf.get("type", "float")]( + name, + min_value=conf["value"][0], + max_value=conf["value"][1], + scale_type=MAP_SCALE_TYPE[conf.get("scale_type", "linear")], + ) + + +def cast_params(params: Dict, config: Dict) -> Dict: + """ + Cast params to integer according to configuration. + + Parameters + ---------- + params : Dict + Dictionary with suggested parameters + config : Dict + Provided configuration with types + + Returns + ------- + Dict + Updated parameters + """ + for key, value in params.items(): + if config[key]["type"] == "int": + params[key] = int(value) + return params + + +def main( + args: argparse.Namespace, + config: Dict, + install_path: Path, + server_endpoint: str = None, +) -> Dict: + """ + Converts config to Vizier problem definition and runs optimization. + + Parameters + ---------- + args : argparse.Namespace + Optimization arguments + config : Dict + Optimization configuration + install_path : Path + Path to the folder with installed ORFS binaries + server_endpoint : str + URL pointing to Vizier server + + Returns + ------- + Dict + Results of optimization, containing 'config', 'population' + and found 'optimals' + """ + results = {"config": config, "populations": [], "optimals": []} + + problem = vz.ProblemStatement() + for key, value in config.items(): + if isinstance(value, Dict): + register_param(args, problem, key, value) + for metric in args.use_metrics: + problem.metric_information.append( + vz.MetricInformation(metric, goal=METRIC_TO_GOAL[metric]) + ) + + study_config = vz.StudyConfig.from_problem(problem) + study_config.algorithm = args.algorithm + + # Vizier Client setup + if server_endpoint: + clients.environment_variables.server_endpoint = server_endpoint + study_client = clients.Study.from_study_config( + study_config, owner="owner", study_id=f"{args.experiment}-{args.design}" + ) + + state = study_client.materialize_state() + start_iteration = 0 + # Check if experiment should be continued + if state == vz.StudyState.COMPLETED or state == vz.StudyState.ABORTED: + trials = list(study_client.trials().get()) + last_iteration = max( + map(lambda x: int(x.metadata.get("iteration", -1)), trials) + ) + start_iteration = last_iteration + 1 + if start_iteration <= args.iterations - 1: + print(f"[WARN TUN-0026] Trying to restart experiment (previously {state})") + study_client.set_state(vz.StudyState.ACTIVE) + + # Run iterations + for i, s in zip( + range(start_iteration, args.iterations), args.suggestions[start_iteration:] + ): + try: + suggestions = study_client.suggest(count=s) + unfinished = [ + parallel_evaluate.remote( + args, + cast_params(suggestion.parameters, config), + i, + s_i, + install_path, + ) + for s_i, suggestion in enumerate(suggestions) + ] + # Setup tqdm + print("\n") # Prepare space for additional info + tqdm_population = tqdm(total=s) + tqdm_population.set_description(f"Iteration {i + 1}/{args.iterations}") + while unfinished: + finished, unfinished = ray.wait(unfinished, num_returns=1) + sample = ray.get(finished)[0] + print(sample) + results["populations"].append(sample) + final_measurement = vz.Measurement(sample["evaluation"]) + process_suggestion = suggestions[sample["suggestion"]] + process_suggestion.update_metadata( + vz.Metadata( + { + "variant": sample["variant"], + "duration": str(sample["duration"]), + "iteration": str(i), + "suggestion": str(s), + } + ) + ) + # Display suggestion's parameters and evaluations + tqdm_population.display( + f"[INFO TUN-0024] Params: {process_suggestion.parameters}\n" + f"Evaluation: {sample['evaluation']}\n", + -1, + ) + tqdm_population.update(1) + suggestions[sample["suggestion"]].complete(final_measurement) + except KeyboardInterrupt as ex: + study_client.set_state(vz.StudyState.ABORTED) + raise ex + + study_client.set_state(vz.StudyState.COMPLETED) + + for optimal_trial in study_client.optimal_trials(): + trial = optimal_trial.materialize() + print( + f"[INFO TUN-0027] Found params: {trial.parameters.as_dict()}\nMetrics: {trial.final_measurement.metrics}" + ) + results["optimals"].append( + { + "params": trial.parameters.as_dict(), + "evaluation": { + k: v.value for k, v in trial.final_measurement.metrics.items() + }, + "variant": f"{args.experiment}/{trial.metadata.get_or_error('variant')}", + "time": float(trial.metadata.get_or_error("duration")), + } + ) + return results + + +def initialize_parser() -> argparse.ArgumentParser: + """ + Creates parser with required arguments. + + Returns + ------- + argparse.ArgumentParser + Preared parser + """ + parser = argparse.ArgumentParser() + add_common_args(parser) + parser.add_argument( + "--experiment", + type=str, + metavar="", + default=f"test-{DATE}", + help="Experiment name. This parameter is used to prefix the" + " FLOW_VARIANT and as the Vizier study ID.", + ) + parser.add_argument( + "--orfs", + type=Path, + metavar="", + default=ORFS, + help="Path to the OpenROAD-flow-scripts repository", + ) + parser.add_argument( + "--results", + type=Path, + metavar="", + default="results.json", + help="Path where JSON file with results will be saved", + ) + parser.add_argument( + "-a", + "--algorithm", + type=str, + choices=[ + "GAUSSIAN_PROCESS_BANDIT", + "RANDOM_SEARCH", + "QUASI_RANDOM_SEARCH", + "GRID_SEARCH", + "SHUFFLED_GRID_SEARCH", + "NSGA2", + ], + help="Algorithm for the optimization engine", + default="NSGA2", + ) + available_metrics = list(METRIC_TO_GOAL.keys()) + parser.add_argument( + "-m", + "--use-metrics", + nargs="+", + choices=available_metrics, + default=available_metrics, + help="Metrics to optimize", + ) + parser.add_argument( + "-i", + "--iterations", + type=int, + metavar="", + help="Max iteration count for the optimization engine", + default=2, + ) + parser.add_argument( + "-s", + "--suggestions", + type=int, + metavar="", + nargs="+", + help="Suggestion count per iteration of the optimization engine", + default=[5], + ) + vizier_server_args = parser.add_mutually_exclusive_group() + vizier_server_args.add_argument( + "--vz-use-existing-server", + type=str, + metavar="", + help="Address of the running Vizier server", + default=None, + ) + start_server_args = vizier_server_args.add_argument_group("Local server") + start_server_args.add_argument( + "--vz-server-host", + type=str, + metavar="", + help="Spawn Vizier server with given host", + default=None, + ) + start_server_args.add_argument( + "--vz-server-db", + type=str, + metavar="", + help="Path to the Vizier server's database", + default=None, + ) + return parser + + +def run_vizier(): + """ + Entrypoint for Vizier optimization. + + Parses arguments and config, prepares Vizier server, + runs optimization and saves the results. + """ + parser = initialize_parser() + args = parser.parse_args() + + if args.algorithm == "GAUSSIAN_PROCESS_BANDIT" and any( + s > 1 for s in args.suggestions + ): + print( + "[ERROR TUN-0022] GAUSSIAN_PROCESS_BANDIT does not support " + "batch operation, please set suggestions to 1", + file=sys.stderr, + ) + exit(1) + + args.results = args.results.absolute() + args.mode = "vizier" + args.suggestions += [ + args.suggestions[-1] for _ in range(args.iterations - len(args.suggestions)) + ] + + config, sdc_file, fr_file = read_config(args.config, "vizier", args.algorithm) + args.sdc_file = sdc_file + args.fr_file = fr_file + + local_dir, orfs_flow_dir, install_path = prepare_ray_server(args) + args.orfs = Path(orfs_flow_dir).parent + + server_endpoint = None + if args.vz_server_host: + # Start Vizier server + server_database = ( + args.vz_server_db if args.vz_server_db else service.SQL_LOCAL_URL + ) + server = servers.DefaultVizierServer( + host=args.vz_server_host, + database_url=server_database, + ) + print(f"[INFO TUN-0020] Started Vizier Server at: {server.endpoint}") + print(f"[INFO TUN-0021] SQL database file located at: {server._database_url}") + server_endpoint = server.endpoint + if args.vz_use_existing_server: + server_endpoint = args.vz_use_existing_server + + results = main(args, config, install_path, server_endpoint) + with args.results.open("w") as fd: + json.dump(results, fd) + print(f"[INFO TUN-0002] Results saved to {args.results}") + + +if __name__ == "__main__": + run_vizier() From 9c32d657a4b854b62cc66ba30490c185fd5729c5 Mon Sep 17 00:00:00 2001 From: Eryk Szpotanski Date: Mon, 21 Oct 2024 15:43:05 +0200 Subject: [PATCH 4/6] tools: Autotuner: Add --to-stage argument Signed-off-by: Eryk Szpotanski --- tools/AutoTuner/src/autotuner/distributed.py | 5 ++- tools/AutoTuner/src/autotuner/utils.py | 44 +++++++++++++++----- tools/AutoTuner/src/autotuner/vizier.py | 2 +- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/tools/AutoTuner/src/autotuner/distributed.py b/tools/AutoTuner/src/autotuner/distributed.py index 8e5295c9ff..5f3be05720 100644 --- a/tools/AutoTuner/src/autotuner/distributed.py +++ b/tools/AutoTuner/src/autotuner/distributed.py @@ -110,10 +110,11 @@ def step(self): parameters=self.parameters, flow_variant=self._variant, install_path=INSTALL_PATH, + stage=args.to_stage, ) self.step_ += 1 (score, effective_clk_period, num_drc) = self.evaluate( - read_metrics(metrics_file) + read_metrics(metrics_file, args.to_stage) ) # Feed the score back to Tune. # return must match 'metric' used in tune.run() @@ -455,7 +456,7 @@ def sweep(): TrainClass = set_training_class(args.eval) # PPAImprov requires a reference file to compute training scores. if args.eval == "ppa-improv": - reference = read_metrics(args.reference) + reference = read_metrics(args.reference, args.to_stage) tune_args = dict( name=args.experiment, diff --git a/tools/AutoTuner/src/autotuner/utils.py b/tools/AutoTuner/src/autotuner/utils.py index d696f28afa..f00fb75b16 100644 --- a/tools/AutoTuner/src/autotuner/utils.py +++ b/tools/AutoTuner/src/autotuner/utils.py @@ -29,6 +29,12 @@ set_input_delay [expr $clk_period * $clk_io_pct] -clock $clk_name $non_clock_inputs set_output_delay [expr $clk_period * $clk_io_pct] -clock $clk_name [all_outputs] """ +# Maps ORFS stage to a name of produced metrics +STAGE_TO_METRICS = { + "route": "detailedroute", + "place": "detailedplace", + "final": "finish", +} # Name of the SDC file with constraints CONSTRAINTS_SDC = "constraint.sdc" # Name of the TCL script run before routing @@ -246,6 +252,7 @@ def openroad( flow_variant, path="", install_path=None, + stage="", ): """ Run OpenROAD-flow-scripts with a given set of parameters. @@ -269,7 +276,7 @@ def openroad( make_command = export_command make_command += f"make -C {base_dir}/flow DESIGN_CONFIG=designs/" - make_command += f"{args.platform}/{args.design}/config.mk" + make_command += f"{args.platform}/{args.design}/config.mk {stage}" make_command += f" PLATFORM={args.platform}" make_command += f" FLOW_VARIANT={flow_variant} {parameters}" make_command += " EQUIVALENCE_CHECK=0" @@ -320,10 +327,11 @@ def openroad( ) -def read_metrics(file_name): +def read_metrics(file_name, stage=""): """ Collects metrics to evaluate the user-defined objective function. """ + metric_name = STAGE_TO_METRICS.get(stage if stage else "final", stage) with open(file_name) as file: data = json.load(file) clk_period = 9999999 @@ -346,17 +354,17 @@ def read_metrics(file_name): num_drc = value["route__drc_errors"] if stage_name == "detailedroute" and "route__wirelength" in value: wirelength = value["route__wirelength"] - if stage_name == "finish" and "timing__setup__ws" in value: + if stage_name == metric_name and "timing__setup__ws" in value: worst_slack = value["timing__setup__ws"] - if stage_name == "finish" and "power__total" in value: + if stage_name == metric_name and "power__total" in value: total_power = value["power__total"] - if stage_name == "finish" and "design__instance__utilization" in value: + if stage_name == metric_name and "design__instance__utilization" in value: final_util = value["design__instance__utilization"] - if stage_name == "finish" and "design__instance__area" in value: + if stage_name == metric_name and "design__instance__area" in value: design_area = value["design__instance__area"] - if stage_name == "finish" and "design__core__area" in value: + if stage_name == metric_name and "design__core__area" in value: core_area = value["design__core__area"] - if stage_name == "finish" and "design__die__area" in value: + if stage_name == metric_name and "design__die__area" in value: die_area = value["design__die__area"] for i, stage_name in reversed(STAGES): if stage_name in data and [d for d in data[stage_name].values() if d != "ERR"]: @@ -371,9 +379,15 @@ def read_metrics(file_name): "design_area": design_area, "core_area": core_area, "die_area": die_area, - "wirelength": wirelength, - "num_drc": num_drc, - } + "last_successful_stage": last_stage, + } | ( + { + "wirelength": wirelength, + "num_drc": num_drc, + } + if metric_name in ("detailedroute", "finish") + else {} + ) return ret @@ -682,6 +696,7 @@ def openroad_distributed( flow_variant=f"{uuid()}-{variant}", path=path, install_path=install_path, + stage=args.to_stage, ) duration = time() - t return metric_file, duration @@ -722,6 +737,13 @@ def add_common_args(parser: argparse.ArgumentParser): required=True, help="Configuration file that sets which knobs to use for Autotuning.", ) + parser.add_argument( + "--to-stage", + type=str, + choices=("floorplan", "place", "cts", "route", "finish"), + default="", + help="Run ORFS only to the given stage (inclusive)", + ) parser.add_argument( "--timeout", type=float, diff --git a/tools/AutoTuner/src/autotuner/vizier.py b/tools/AutoTuner/src/autotuner/vizier.py index 225df742ed..56edd3f1a1 100644 --- a/tools/AutoTuner/src/autotuner/vizier.py +++ b/tools/AutoTuner/src/autotuner/vizier.py @@ -64,7 +64,7 @@ def evaluate(args: argparse.Namespace, metric_file: str) -> Dict[str, float]: Dictionary with metrics """ try: - metrics = read_metrics(metric_file) + metrics = read_metrics(metric_file, stage=args.to_stage) # Calculate difference of clock period and worst slack if metrics["clk_period"] != 9999999 and metrics["worst_slack"] != "ERR": metrics["clk_period-worst_slack"] = ( From e218cdd2811a7cf2edf5febd2023dfa9d00f6ebb Mon Sep 17 00:00:00 2001 From: Eryk Szpotanski Date: Mon, 21 Oct 2024 15:44:15 +0200 Subject: [PATCH 5/6] tools: Autotuner: Add smoke test for Vizier Signed-off-by: Eryk Szpotanski --- flow/test/test_autotuner.sh | 3 ++ tools/AutoTuner/test/smoke_test_vizier.py | 52 +++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 tools/AutoTuner/test/smoke_test_vizier.py diff --git a/flow/test/test_autotuner.sh b/flow/test/test_autotuner.sh index 7f8c258753..73abd8976c 100755 --- a/flow/test/test_autotuner.sh +++ b/flow/test/test_autotuner.sh @@ -22,6 +22,9 @@ python3 -m unittest tools.AutoTuner.test.smoke_test_sweep.${PLATFORM}SweepSmokeT echo "Running Autotuner smoke tests for --sample and --iteration." python3 -m unittest tools.AutoTuner.test.smoke_test_sample_iteration.${PLATFORM}SampleIterationSmokeTest.test_sample_iteration +echo "Running Autotuner smoke Vizier test" +python3 -m unittest tools.AutoTuner.test.smoke_test_vizier.${PLATFORM}VizierSmokeTest.test_vizier + if [ "$PLATFORM" == "asap7" ] && [ "$DESIGN_NAME" == "gcd" ]; then echo "Running Autotuner ref file test (only once)" python3 -m unittest tools.AutoTuner.test.ref_file_check.RefFileCheck.test_files diff --git a/tools/AutoTuner/test/smoke_test_vizier.py b/tools/AutoTuner/test/smoke_test_vizier.py new file mode 100644 index 0000000000..648d2229d4 --- /dev/null +++ b/tools/AutoTuner/test/smoke_test_vizier.py @@ -0,0 +1,52 @@ +import unittest +import subprocess +import os +from datetime import datetime + +cur_dir = os.path.dirname(os.path.abspath(__file__)) + + +class BaseVizierSmokeTest(unittest.TestCase): + platform = "" + design = "" + + def setUp(self): + self.config = os.path.join( + cur_dir, + f"../../../flow/designs/{self.platform}/{self.design}/autotuner.json", + ) + self.experiment = f"smoke-test-tune-{self.platform}-{datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}" + self.command = ( + "python3 -m autotuner.vizier" + f" --design {self.design}" + f" --platform {self.platform}" + f" --experiment {self.experiment}" + f" --config {self.config}" + f" --iteration 1 --suggestions 1" + ) + + def test_vizier(self): + if not (self.platform and self.design): + raise unittest.SkipTest("Platform and design have to be defined") + out = subprocess.run(self.command, shell=True, check=True) + successful = out.returncode == 0 + self.assertTrue(successful) + + +class ASAP7VizierSmokeTest(BaseVizierSmokeTest): + platform = "asap7" + design = "gcd" + + +class SKY130HDVizierSmokeTest(BaseVizierSmokeTest): + platform = "sky130hd" + design = "gcd" + + +class IHPSG13G2VizierSmokeTest(BaseVizierSmokeTest): + platform = "ihp-sg13g2" + design = "gcd" + + +if __name__ == "__main__": + unittest.main() From 5f72b6f8ce2b9e7d1065072c7354549b151e2dba Mon Sep 17 00:00:00 2001 From: Eryk Szpotanski Date: Mon, 21 Oct 2024 15:46:26 +0200 Subject: [PATCH 6/6] docs: user: Add information about Vizier in AutoTuner Signed-off-by: Eryk Szpotanski --- docs/user/InstructionsForAutoTuner.md | 91 ++++++++++++++++++++------- 1 file changed, 70 insertions(+), 21 deletions(-) diff --git a/docs/user/InstructionsForAutoTuner.md b/docs/user/InstructionsForAutoTuner.md index e5ad9874d4..db6eb09ad3 100644 --- a/docs/user/InstructionsForAutoTuner.md +++ b/docs/user/InstructionsForAutoTuner.md @@ -5,20 +5,29 @@ AutoTuner provides a generic interface where users can define parameter configur This enables AutoTuner to easily support various tools and flows. AutoTuner also utilizes [METRICS2.1](https://github.com/ieee-ceda-datc/datc-rdf-Metrics4ML) to capture PPA of individual search trials. With the abundant features of METRICS2.1, users can explore various reward functions that steer the flow autotuning to different PPA goals. -AutoTuner provides two main functionalities as follows. -* Automatic hyperparameter tuning framework for OpenROAD-flow-script (ORFS) -* Parametric sweeping experiments for ORFS +AutoTuner provides three main functionalities as follows. +* [Ray] Automatic hyperparameter tuning framework for OpenROAD-flow-script (ORFS) +* [Ray] Parametric sweeping experiments for ORFS +* [Vizier] Multi-objective optimization of ORFS parameters AutoTuner contains top-level Python script for ORFS, each of which implements a different search algorithm. Current supported search algorithms are as follows. -* Random/Grid Search -* Population Based Training ([PBT](https://www.deepmind.com/blog/population-based-training-of-neural-networks)) -* Tree Parzen Estimator ([HyperOpt](https://hyperopt.github.io/hyperopt)) -* Bayesian + Multi-Armed Bandit ([AxSearch](https://ax.dev/)) -* Tree Parzen Estimator + Covariance Matrix Adaptation Evolution Strategy ([Optuna](https://optuna.org/)) -* Evolutionary Algorithm ([Nevergrad](https://github.com/facebookresearch/nevergrad)) +* Ray (Single-objective optimization) + * Random/Grid Search + * Population Based Training ([PBT](https://www.deepmind.com/blog/population-based-training-of-neural-networks)) + * Tree Parzen Estimator ([HyperOpt](https://hyperopt.github.io/hyperopt)) + * Bayesian + Multi-Armed Bandit ([AxSearch](https://ax.dev/docs/bayesopt.html)) + * Tree Parzen Estimator + Covariance Matrix Adaptation Evolution Strategy ([Optuna](https://optuna.readthedocs.io/en/stable/reference/samplers/generated/optuna.samplers.TPESampler.html)) + * Evolutionary Algorithm ([Nevergrad](https://github.com/facebookresearch/nevergrad)) +* Vizier (Multi-objective optimization) + * Random/Grid/Shuffled Search + * Quasi Random Search ([quasi-random](https://developers.google.com/machine-learning/guides/deep-learning-tuning-playbook/quasi-random-search)) + * Gaussian Process Bandit ([GP-Bandit](https://acsweb.ucsd.edu/~shshekha/GPBandits.html)) + * Non-dominated Sorting Genetic Algorithm II ([NSGA-II](https://ieeexplore.ieee.org/document/996017)) -User-defined coefficient values (`coeff_perform`, `coeff_power`, `coeff_area`) of three objectives to set the direction of tuning are written in the script. Each coefficient is expressed as a global variable at the `get_ppa` function in `PPAImprov` class in the script (`coeff_perform`, `coeff_power`, `coeff_area`). Efforts to optimize each of the objectives are proportional to the specified coefficients. +For Ray algorithms, optimized function can be adjusted with user-defined coefficient values (`coeff_perform`, `coeff_power`, `coeff_area`) for three objectives to set the direction of tuning. They are defined in the [distributed.py sricpt](../../tools/AutoTuner/src/autotuner/distributed.py) in `get_ppa` method of `PPAImprov` class. Efforts to optimize each of the objectives are proportional to the specified coefficients. + +Using Vizier algorithms, used can choose which metrics should be optimized with `--use-metrics` argument. ## Setting up AutoTuner @@ -28,8 +37,10 @@ that works in Python3.8 for installation and configuration of AutoTuner, as shown below: ```shell -# Install prerequisites +# Install prerequisites for both Ray Tune and Vizier ./tools/AutoTuner/installer.sh +# Or install prerequisites for `ray` or `vizier` +./tools/AutoTuner/installer.sh vizier # Start virtual environment ./tools/AutoTuner/setup.sh @@ -50,7 +61,8 @@ Alternatively, here is a minimal example to get started: 1.0, 3.7439 ], - "step": 0 + "step": 0, + "scale": "log" }, "CORE_MARGIN": { "type": "int", @@ -67,6 +79,7 @@ Alternatively, here is a minimal example to get started: * `"type"`: Parameter type ("float" or "int") for sweeping/tuning * `"minmax"`: Min-to-max range for sweeping/tuning. The unit follows the default value of each technology std cell library. * `"step"`: Parameter step within the minmax range. Step 0 for type "float" means continuous step for sweeping/tuning. Step 0 for type "int" means the constant parameter. +* `"scale"`: Vizier-specific parameter setting [scaling type](https://oss-vizier.readthedocs.io/en/latest/guides/user/search_spaces.html#scaling), allowed values: `linear`, `log` and `rlog`. ## Tunable / sweepable parameters @@ -118,13 +131,21 @@ The order of the parameters matter. Arguments `--design`, `--platform` and `--config` are always required and should precede *mode*. ``` +The `autotuner.vizier` module integrates OpenROAD flow into the Vizier optimizer. +It is used for multi-objective optimization with an additional features improving chance of finding valid parameters. +Moreover, various algorithms are available for tuning parameters. + +Each mode relies on user-specified search space that is +defined by a `.json` file, they use the same syntax and format, +though some features may not be available for sweeping. + ```{note} The following commands should be run from `./tools/AutoTuner`. ``` #### Tune only -* AutoTuner: `python3 -m autotuner.distributed tune -h` +* Ray-based AutoTuner: `python3 -m autotuner.distributed tune -h` Example: @@ -145,19 +166,39 @@ python3 -m autotuner.distributed --design gcd --platform sky130hd \ sweep ``` +#### Multi-object optimization + +* Vizier-based AutoTuner: `python3 -m autotuner.vizier -h` + +Example: + +```shell +python3 -m autotuner.vizier --design gcd --platform sky130hd \ + --config ../../flow/designs/sky130hd/gcd/autotuner.json +``` ### Google Cloud Platform (GCP) distribution with Ray GCP Setup Tutorial coming soon. -### List of input arguments +### List of common input arguments | Argument | Description | Default | |-------------------------------|-------------------------------------------------------------------------------------------------------|---------| | `--design` | Name of the design for Autotuning. || | `--platform` | Name of the platform for Autotuning. || | `--config` | Configuration file that sets which knobs to use for Autotuning. || | `--experiment` | Experiment name. This parameter is used to prefix the FLOW_VARIANT and to set the Ray log destination.| test | +| `--samples` | Number of samples for tuning. | 10 | +| `--jobs` | Max number of concurrent jobs. | # of CPUs / 2 | +| `--openroad_threads` | Max number of threads usable. | 16 | +| `--timeout` | Time limit (in hours) for each trial run. | No limit | +| `-v` or `--verbose` | Verbosity Level. [0: Only ray status, 1: print stderr, 2: print stdout on top of what is in level 0 and 1. ] | 0 | +| | || + +### Input arguments specific to Ray +| Argument | Description | Default | +|-------------------------------|-------------------------------------------------------------------------------------------------------|---------| | `--git_clean` | Clean binaries and build files. **WARNING**: may lose previous data. || | `--git_clone` | Force new git clone. **WARNING**: may lose previous data. || | `--git_clone_args` | Additional git clone arguments. || @@ -166,16 +207,11 @@ GCP Setup Tutorial coming soon. | `--git_orfs_branch` | OpenROAD-flow-scripts branch to use. || | `--git_url` | OpenROAD-flow-scripts repo URL to use. | [ORFS GitHub repo](https://github.com/The-OpenROAD-Project/OpenROAD-flow-scripts) | | `--build_args` | Additional arguments given to ./build_openroad.sh || -| `--samples` | Number of samples for tuning. | 10 | -| `--jobs` | Max number of concurrent jobs. | # of CPUs / 2 | -| `--openroad_threads` | Max number of threads usable. | 16 | | `--server` | The address of Ray server to connect. || | `--port` | The port of Ray server to connect. | 10001 | -| `--timeout` | Time limit (in hours) for each trial run. | No limit | -| `-v` or `--verbose` | Verbosity Level. [0: Only ray status, 1: print stderr, 2: print stdout on top of what is in level 0 and 1. ] | 0 | | | || -#### Input arguments specific to tune mode +#### Input arguments specific to Ray tune mode The following input arguments are applicable for tune mode only. | Argument | Description | Default | @@ -190,7 +226,19 @@ The following input arguments are applicable for tune mode only. | `--resume` | Resume previous run. || | | || -### GUI +### Input arguments specific to Vizier +| Argument | Description | Default | +|-------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| +| `--orfs` | Path to the OpenROAD-flow-scripts repository || +| `--results` | Path where JSON file with results will be saved || +| `-a` or `--algorithm` | Algorithm for the optimization engine, one of GAUSSIAN_PROCESS_BANDIT, RANDOM_SEARCH, QUASI_RANDOM_SEARCH, GRID_SEARCH, SHUFFLED_GRID_SEARCH, NSGA2 | NSGA2 | +| `-m` or `--use-metrics` | Metrics to optimize, list of worst_slack, clk_period-worst_slack, total_power, core_util, final_util, design_area, core_area, die_area, last_successful_stage | all available metrics | +| `-i` or `--iterations` | Max iteration count for the optimization engine | 2 || +| `-s` or `--suggestions` | Suggestion count per iteration of the optimization engine | 5 || +| `--use-existing-server` | Address of the running Vizier server || +| | || + +### GUI for optimizations with Ray Tune Basically, progress is displayed at the terminal where you run, and when all runs are finished, the results are displayed. You could find the "Best config found" on the screen. @@ -216,6 +264,7 @@ Assuming the virtual environment is setup at `./tools/AutoTuner/autotuner_env`: ./tools/AutoTuner/setup.sh python3 ./tools/AutoTuner/test/smoke_test_sweep.py python3 ./tools/AutoTuner/test/smoke_test_tune.py +python3 ./tools/AutoTuner/test/smoke_test_vizier.py python3 ./tools/AutoTuner/test/smoke_test_sample_iteration.py ```