diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index bb5e1d3a0..83edafdf1 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -222,10 +222,10 @@ jobs: pip install -r ./requirements.txt - name: Run run: |- - output=$(./bin/benchpark configure --bootstrap-location . && ./bin/benchpark bootstrap 2>&1) + output=$(./bin/benchpark configure --bootstrap-location ./bootstrap && ./bin/benchpark bootstrap 2>&1) # 1. Check the path is present - echo "$output" | grep -q "/home/runner/work/benchpark/benchpark/.benchpark/ramble" \ + echo "$output" | grep -q "/home/runner/work/benchpark/benchpark/bootstrap/ramble" \ || { echo "Expected path not found"; exit 1; } # 2. Check "Cloning Ramble" appears exactly once diff --git a/config/bootstrap.yaml b/config/bootstrap.yaml new file mode 100644 index 000000000..4211d0e9e --- /dev/null +++ b/config/bootstrap.yaml @@ -0,0 +1,6 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 +bootstrap: + location: ~/.benchpark/ diff --git a/config/repos.yaml b/config/repos.yaml new file mode 100644 index 000000000..2dffe1aeb --- /dev/null +++ b/config/repos.yaml @@ -0,0 +1,9 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 +repos: + experiments: [../experiments, ../experiments/test] + systems: [../systems] + applications: [../repo] + packages: [../repo] diff --git a/docs/configuration.rst b/docs/configuration.rst new file mode 100644 index 000000000..54312256b --- /dev/null +++ b/docs/configuration.rst @@ -0,0 +1,68 @@ +.. + Copyright 2023 Lawrence Livermore National Security, LLC and other + Benchpark Project Developers. See the top-level COPYRIGHT file for details. + + SPDX-License-Identifier: Apache-2.0 + +####################### + Configuring Benchpark +####################### + +Benchpark offers several options to configure usage. This includes: + +1. Configuring the location of the bootstrap directory (default is ``~/.benchpark``). + This is useful to change from the default value if you do not have much space in your + home directory, as the bootstrap contains the ``spack``, ``spack-packages``, and + ``ramble`` repositories. + +1. Configuring which repositories benchpark uses for objects in the + ``experiments/systems/repos`` directories (``experiment.py``, ``system.py``, + ``package.py``, and ``application.py``). + +********************************************** + Configuring the Benchpark Bootstrap Location +********************************************** + +Benchpark clones ``ramble``, ``spack``, and ``spack-packages`` into a centralized +location (by default this is ``~/.benchpark``) to enable building and running +benchmarks. Benchpark workspaces copy over these repositories from the centralized +location to avoid downloading from the internet when initializing a Benchpark workspace. +The bootstrap location can be configured in ``/config/bootstrap.yaml`` +or by running ``benchpark configure --bootstrap-location ``. + +*********************************************** + Configuring Which Repositories Benchpark Uses +*********************************************** + +The ``/config/repos.yaml`` file is used to fully customize ``system`` +and ``experiment`` repositories used by Benchpark, ``application`` repositories used by +Ramble, and package repositories used by Spack. For Spack and Ramble, the builtin +repository is always used (in addition to whatever package repositories are specified). +The order in which repositories appear in ``/config/repos.yaml`` will +determine their precedence, so if you desire to use a custom repository, it should be +listed before any other repositories. + +********************************************************************** + Automatically Generating Configurations with ``benchpark configure`` +********************************************************************** + +``benchpark configure`` is designed to create the ``yaml`` configurations for you. As of +now, it can only generate bootstrap config (``bootstrap.yaml``). If no bootstrap config +is detected in the chosen scope, this will auto-generate it. ``benchpark configure`` +cannot generate a ``repos.yaml`` file at the moment, this must be written manually. + +********************** + Configuration Scopes +********************** + +Benchpark can pull configurations from one location, with the following priority +(highest first): + +- As an argument to ``benchpark``: ``benchpark -C ...`` +- If the CWD where you invoke the benchpark executable has ``benchpark-config`` + directory +- If ``/config`` is a directory; this must exist if the first two don't + +There is no mixing and matching between these tiers. If you are using ``-C``, then the +specified directory must contain a ``bootstrap.yaml`` and a ``repos.yaml`` (you can copy +from ``/config`` if you want the same config). diff --git a/docs/generate-benchmark-list.py b/docs/generate-benchmark-list.py index f6bc005be..852bd0252 100755 --- a/docs/generate-benchmark-list.py +++ b/docs/generate-benchmark-list.py @@ -8,7 +8,6 @@ sys.path.append("../lib/") import benchpark.accounting # noqa: E402 -import benchpark.paths # noqa: E402 def construct_tag_groups(tag_groups, tag_dicts, dictionary): diff --git a/docs/index.rst b/docs/index.rst index 03c879b84..afdd5147e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -40,6 +40,7 @@ run-experiment analyze-experiment benchpark-analyze + configuration modifiers set-of-experiments run-binary diff --git a/experiments/test/repo.yaml b/experiments/test/repo.yaml new file mode 100644 index 000000000..6f321df87 --- /dev/null +++ b/experiments/test/repo.yaml @@ -0,0 +1,7 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 +repo: + namespace: test + subdirectory: '' diff --git a/experiments/saxpy/experiment.py b/experiments/test/saxpy/experiment.py similarity index 100% rename from experiments/saxpy/experiment.py rename to experiments/test/saxpy/experiment.py diff --git a/lib/benchpark/accounting.py b/lib/benchpark/accounting.py index eb31c5a7c..e8a289fee 100644 --- a/lib/benchpark/accounting.py +++ b/lib/benchpark/accounting.py @@ -6,7 +6,7 @@ import os from collections import defaultdict -import benchpark.paths +from benchpark.base_paths import base_paths PROGRAMMING_MODEL_CATEGORY = "programming_model" SCALING_CATEGORY = "scaling" @@ -30,7 +30,7 @@ def benchpark_experiments(exp_dict=EXP_DICT, exclude_variants=[]): - source_dir = benchpark.paths.benchpark_root + source_dir = base_paths.benchpark_root experiments = [] experiments_dir = source_dir / "experiments" @@ -60,7 +60,7 @@ def benchpark_experiments(exp_dict=EXP_DICT, exclude_variants=[]): def benchpark_modifiers(): all_benchmark_modifiers = ["affinity", "allocation", "hwloc"] - source_dir = benchpark.paths.benchpark_root + source_dir = base_paths.benchpark_root experiments_dir = source_dir / "experiments" modifiers = [] exclude = ["modifier_repo.yaml"] @@ -88,7 +88,7 @@ def benchpark_modifiers(): def benchpark_systems(programming_model=None): from benchpark.spec import SystemSpec - source_dir = benchpark.paths.benchpark_root + source_dir = base_paths.benchpark_root systems = [] exclude = ["all_hardware_descriptions", "common", "repo.yaml"] for x in sorted(os.listdir(source_dir / "systems")): @@ -128,7 +128,7 @@ def benchpark_systems(programming_model=None): def benchpark_benchmarks(): - source_dir = benchpark.paths.benchpark_root + source_dir = base_paths.benchpark_root benchmarks = [] experiments_dir = source_dir / "experiments" for x in sorted(os.listdir(experiments_dir)): diff --git a/lib/benchpark/base_paths.py b/lib/benchpark/base_paths.py new file mode 100644 index 000000000..0ce4010de --- /dev/null +++ b/lib/benchpark/base_paths.py @@ -0,0 +1,56 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +import pathlib + + +def _source_location() -> pathlib.Path: + """Return the location of the project source files directory.""" + path_to_this_file = __file__ + return pathlib.Path(path_to_this_file).resolve().parents[2] + + +_unset = object() + + +class BasePaths: + def __init__(self): + self.benchpark_root = _source_location() + self.lib_path = self.benchpark_root / "lib" / "benchpark" + self.test_path = self.lib_path / "test" + self.hardware_descriptions = ( + self.benchpark_root / "systems" / "all_hardware_descriptions" + ) + self.checkout_versions = self.benchpark_root / "checkout-versions.yaml" + self.remote_urls = self.benchpark_root / "remote-urls.yaml" + self.invocation_working_dir = None + self.user_input_cfg = _unset + + +base_paths = BasePaths() + + +def determine_config_dir(): + """ + Benchpark configs don't merge or override like Spack/Ramble. You + just point it at a directory and that's where all your config is. + """ + if base_paths.user_input_cfg is _unset: + raise Exception("Internal error: config initialization") + elif base_paths.user_input_cfg: + if not base_paths.user_input_cfg.exists(): + raise Exception( + f"Specific config dir does not exist: {base_paths.user_input_cfg}" + ) + else: + return base_paths.user_input_cfg + + possible_dirs = [ + base_paths.invocation_working_dir / "benchpark-config", + base_paths.benchpark_root / "config", + ] + for pd in possible_dirs: + if pd.exists(): + return pd diff --git a/lib/benchpark/cmd/configure.py b/lib/benchpark/cmd/configure.py index 4020b07d0..6a1ccac5d 100644 --- a/lib/benchpark/cmd/configure.py +++ b/lib/benchpark/cmd/configure.py @@ -8,7 +8,7 @@ import yaml -import benchpark.paths +import benchpark.config def setup_parser(root_parser): @@ -21,15 +21,14 @@ def setup_parser(root_parser): def command(args): - data = {} - - if args.bootstrap_location: - loc = os.path.expandvars(os.path.expanduser(args.bootstrap_location)) - bl = str(Path(loc).resolve()).rstrip("/") + "/.benchpark/" - data["bootstrap"] = { - "location": bl, - } - - print(f"Writing configuration to {benchpark.paths.benchpark_config}") - with open(benchpark.paths.benchpark_config, "w") as yaml_file: - yaml.safe_dump(data, yaml_file) + bootstrap_cfg = benchpark.config.configuration().bootstrap + + if args.bootstrap_location or not bootstrap_cfg.path.exists(): + where = args.bootstrap_location or "~/.benchpark/" + loc = os.path.expandvars(os.path.expanduser(where)) + bl = str(Path(loc).resolve()).rstrip("/") + data = {"bootstrap": {"location": bl}} + + print(f"Writing configuration to {bootstrap_cfg.path}") + with open(bootstrap_cfg.path, "w") as yaml_file: + yaml.safe_dump(data, yaml_file) diff --git a/lib/benchpark/cmd/info.py b/lib/benchpark/cmd/info.py index f4dc847fd..8be1963f8 100644 --- a/lib/benchpark/cmd/info.py +++ b/lib/benchpark/cmd/info.py @@ -5,7 +5,8 @@ import llnl.util.tty.color as color import yaml -import benchpark.paths +import benchpark.spec +from benchpark.paths import paths def indent(): @@ -71,7 +72,7 @@ def _handle_query(query): key, value = query.split("=", 1) exclude = {"all_hardware_descriptions", "repo.yaml", "generic-x86"} all_system_specs = [] - for d in set(os.listdir(benchpark.paths.benchpark_root / "systems")) - exclude: + for d in set(os.listdir(paths.benchpark_root / "systems")) - exclude: all_system_specs.append(benchpark.spec.SystemSpec(d)) matching_systems = [] diff --git a/lib/benchpark/cmd/mirror.py b/lib/benchpark/cmd/mirror.py index 0e77b3de8..ef3d66988 100644 --- a/lib/benchpark/cmd/mirror.py +++ b/lib/benchpark/cmd/mirror.py @@ -12,7 +12,7 @@ import shutil import tempfile -import benchpark.paths +from benchpark.paths import paths from benchpark.runtime import run_command, working_dir @@ -115,7 +115,7 @@ def mirror_create(args): ) cache_storage = os.path.join(dest, "pip-cache") - ramble_pip_reqs = os.path.join(benchpark.paths.benchpark_root, "requirements.txt") + ramble_pip_reqs = os.path.join(paths.benchpark_root, "requirements.txt") if not os.path.exists(cache_storage): run_command(f"pip download -r {ramble_pip_reqs} -d {cache_storage}") @@ -159,7 +159,7 @@ def _ignore(path, dir_list): env_dir = os.path.dirname(find_one(ramble_workspace, "spack.yaml")) git_repo_dst = os.path.join(dest, "git-repos") repo_copy_script = os.path.join( - benchpark.paths.benchpark_root, "lib", "scripts", "env-collect-branch-tips.py" + paths.benchpark_root, "lib", "scripts", "env-collect-branch-tips.py" ) out, err = run_command( f"spack -e {env_dir} python {repo_copy_script} {git_repo_dst}" @@ -199,12 +199,12 @@ def _ignore(path, dir_list): """) modifiers_dest = os.path.join(dest, "modifiers") - modifiers_src = os.path.join(benchpark.paths.benchpark_root, "modifiers") + modifiers_src = os.path.join(paths.benchpark_root, "modifiers") if not os.path.exists(modifiers_dest): shutil.copytree(modifiers_src, modifiers_dest) bp_repo_dest = os.path.join(dest, "repo") - bp_repo_src = os.path.join(benchpark.paths.benchpark_root, "repo") + bp_repo_src = os.path.join(paths.benchpark_root, "repo") if not os.path.exists(bp_repo_dest): shutil.copytree(bp_repo_src, bp_repo_dest) diff --git a/lib/benchpark/cmd/setup.py b/lib/benchpark/cmd/setup.py index aaa9ba024..fdfa82f6f 100644 --- a/lib/benchpark/cmd/setup.py +++ b/lib/benchpark/cmd/setup.py @@ -12,8 +12,9 @@ import ruamel.yaml as yaml -import benchpark.paths +import benchpark.config from benchpark.debug import debug_print +from benchpark.paths import paths from benchpark.runtime import RuntimeResources @@ -95,7 +96,7 @@ def _find(d, tag): return result experiments_root = pathlib.Path(os.path.abspath(args.experiments_root)) - source_dir = benchpark.paths.benchpark_root + source_dir = paths.benchpark_root experiment_src_dir = pathlib.Path(os.path.abspath(str(args.experiment))) @@ -200,9 +201,11 @@ def include_fn(fname): run_script = experiments_root / ".latest-experiment.sh" per_workspace_setup = RuntimeResources( - experiments_root, upstream=RuntimeResources(benchpark.paths.benchpark_home) + experiments_root, upstream=RuntimeResources(paths.benchpark_home) ) + repos_cfg = benchpark.config.configuration().repos + pkg_str = "" if pkg_manager == "spack": spack_build_stage = experiments_root / "builds" @@ -212,12 +215,18 @@ def include_fn(fname): site_repos = ( per_workspace_setup.spack_location / "etc" / "spack" / "repos.yaml" ) + repos = {} + for repo_dir in reversed(repos_cfg.packages): + repo_dir = repos_cfg.resolve_path(repo_dir) + with open(repo_dir / "repo.yaml", "r") as f: + repo_data = yaml.safe_load(f) + namespace = repo_data["repo"]["namespace"] + repos[namespace] = str(repo_dir) + repos["builtin"] = ( + f"{per_workspace_setup.pkgs_location}/repos/spack_repo/builtin/" + ) with open(site_repos, "w") as f: - f.write(f"""\ -repos:: - benchpark: {source_dir}/repo - builtin: {per_workspace_setup.pkgs_location}/repos/spack_repo/builtin/ -""") + yaml.dump({"repos:": repos}, f, default_flow_style=False) spack( f"config --scope=site add \"config:build_stage:['{spack_build_stage}']\"" ) @@ -230,7 +239,9 @@ def include_fn(fname): ramble, first_time_ramble = per_workspace_setup.ramble_first_time_setup() if first_time_ramble: - ramble(f"repo add --scope=site {source_dir}/repo") + for repo_dir in reversed(repos_cfg.applications): + repo_dir = repos_cfg.resolve_path(repo_dir) + ramble(f"repo add --scope=site {repo_dir}") ramble('config --scope=site add "config:disable_progress_bar:true"') ramble(f"repo add -t modifiers --scope=site {source_dir}/modifiers") ramble("config --scope=site add \"config:spack:global:args:'-d'\"") diff --git a/lib/benchpark/cmd/show_build.py b/lib/benchpark/cmd/show_build.py index fdb947a48..40649e09d 100644 --- a/lib/benchpark/cmd/show_build.py +++ b/lib/benchpark/cmd/show_build.py @@ -11,7 +11,7 @@ import re import shutil -import benchpark.paths +from benchpark.paths import paths from benchpark.runtime import run_command @@ -48,7 +48,7 @@ def show_build_dump(args): env_root = _find_env_root(args.workspace) determine_exp = os.path.join( - benchpark.paths.benchpark_root, "lib", "scripts", "experiment-build-info.py" + paths.benchpark_root, "lib", "scripts", "experiment-build-info.py" ) out, err = run_command(f"spack -e {env_root} python {determine_exp}") exp_info = json.loads(out) diff --git a/lib/benchpark/cmd/system.py b/lib/benchpark/cmd/system.py index ac1e60f31..7549b6d4f 100644 --- a/lib/benchpark/cmd/system.py +++ b/lib/benchpark/cmd/system.py @@ -16,9 +16,9 @@ import yaml from deepdiff import DeepDiff -import benchpark.paths import benchpark.spec import benchpark.system +from benchpark.paths import paths def system_init(args): @@ -63,16 +63,14 @@ def system_external(args): if args.new_system: subprocess.run( [ - benchpark.paths.benchpark_home / "spack/bin/spack", + paths.benchpark_home / "spack/bin/spack", "external", "find", "--not-buildable", ] ) - with open( - benchpark.paths.benchpark_home / "spack/etc/spack/packages.yaml", "r" - ) as file: + with open(paths.benchpark_home / "spack/etc/spack/packages.yaml", "r") as file: new_packages = yaml.safe_load(file)["packages"] color.cprint("@*rHere are all of the new packages:@.") @@ -86,7 +84,7 @@ def system_external(args): pkg_list = list(packages.keys()) subprocess.run( [ - benchpark.paths.benchpark_home / "spack/bin/spack", + paths.benchpark_home / "spack/bin/spack", "external", "find", "--not-buildable", @@ -94,9 +92,7 @@ def system_external(args): + [pkg for pkg in pkg_list] ) - with open( - benchpark.paths.benchpark_home / "spack/etc/spack/packages.yaml", "r" - ) as file: + with open(paths.benchpark_home / "spack/etc/spack/packages.yaml", "r") as file: new_packages = yaml.safe_load(file)["packages"] # Use DeepDiff to find differences diff --git a/lib/benchpark/cmd/unit_test.py b/lib/benchpark/cmd/unit_test.py index 5e1ecc241..e01a5c086 100644 --- a/lib/benchpark/cmd/unit_test.py +++ b/lib/benchpark/cmd/unit_test.py @@ -14,7 +14,7 @@ import llnl.util.tty.color as color import pytest -import benchpark.paths +from benchpark.paths import paths def setup_parser(subparser): @@ -111,10 +111,10 @@ def colorize(c, prefix): # which doesn't need to wait for pytest collection and doesn't # require parsing pytest output files = llnl.util.filesystem.find( - root=benchpark.paths.test_path, files="*.py", recursive=True + root=paths.test_path, files="*.py", recursive=True ) files = [ - os.path.relpath(f, start=benchpark.paths.benchpark_root) + os.path.relpath(f, start=paths.benchpark_root) for f in files if not f.endswith(("conftest.py", "__init__.py")) ] @@ -145,7 +145,7 @@ def colorize(c, prefix): len_indent = len(indent) if os.path.isabs(name): - name = os.path.relpath(name, start=benchpark.paths.benchpark_root) + name = os.path.relpath(name, start=paths.benchpark_root) item = (len_indent, name, nodetype) @@ -212,7 +212,7 @@ def command(args, unknown_args): # add back any parsed pytest args we need to pass to pytest pytest_args = add_back_pytest_args(args, unknown_args) - pytest_root = benchpark.paths.benchpark_root + pytest_root = paths.benchpark_root if args.numprocesses is not None and args.numprocesses > 1: pytest_args.extend( diff --git a/lib/benchpark/config.py b/lib/benchpark/config.py new file mode 100644 index 000000000..86fcda30e --- /dev/null +++ b/lib/benchpark/config.py @@ -0,0 +1,102 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# Copyright 2022-2024 The Ramble Authors +# +# SPDX-License-Identifier: Apache-2.0 + +import pathlib + +import yaml + +import benchpark.base_paths + + +class RequiredClassAttr: + def __init__(self, name): + self.name = name + + def __get__(self, obj, owner): + raise NotImplementedError( + f"{owner.__name__} must define class attribute '{self.name}'" + ) + + +class PropertyDict: + def __getattr__(self, name): + val = self.data[name] + if isinstance(val, dict): + return PropertyDict(val) + return val + + +class ConfigSection(PropertyDict): + def __init__(self, data, path): + self.data = data + self.path = pathlib.Path(path) + + filename = RequiredClassAttr("filename") + name = RequiredClassAttr("name") + + @classmethod + def try_load(cls, cfg_dir): + cfg_path = pathlib.Path(cfg_dir) / cls.filename + if cfg_path.exists(): + with open(cfg_path, "r") as f: + data = yaml.safe_load(f)[cls.name] + else: + data = {} + return cls(data, cfg_path) + + def resolve_path(self, value): + path = pathlib.Path(value) + if not path.is_absolute(): + return (self.path.parents[0] / path).resolve() + + def __getattr__(self, name): + if not self.data: + raise Exception(f"Missing config in {self.path}") + return super().__getattr__(name) + + +class Repos(ConfigSection): + filename = "repos.yaml" + name = "repos" + + +class Bootstrap(ConfigSection): + filename = "bootstrap.yaml" + name = "bootstrap" + + +_section_types = [Repos, Bootstrap] + + +class Configuration: + section_names = [st.name for st in _section_types] + + def __init__(self, cfg_dir): + self.sections = {} + for st in _section_types: + attempt = st.try_load(cfg_dir) + if attempt: + self.sections[st.name] = attempt + + def __getattr__(self, name): + if name in self.sections: + return self.sections[name] + elif name in Configuration.section_names: + raise Exception("This section is not present in this config") + else: + raise AttributeError("No such section") + + +_configuration = None + + +def configuration(): + global _configuration + if not _configuration: + _configuration = Configuration(benchpark.base_paths.determine_config_dir()) + + return _configuration diff --git a/lib/benchpark/directives.py b/lib/benchpark/directives.py index 405c841bf..3965fea55 100644 --- a/lib/benchpark/directives.py +++ b/lib/benchpark/directives.py @@ -18,7 +18,6 @@ import ramble.language.shared_language from ramble.language.language_base import DirectiveError -import benchpark.paths import benchpark.repo import benchpark.runtime import benchpark.spec diff --git a/lib/benchpark/paths.py b/lib/benchpark/paths.py index b172d72c6..467cf9f2f 100644 --- a/lib/benchpark/paths.py +++ b/lib/benchpark/paths.py @@ -6,32 +6,23 @@ import os import pathlib -import yaml +import benchpark.base_paths +import benchpark.config -def _source_location() -> pathlib.Path: - """Return the location of the project source files directory.""" - path_to_this_file = __file__ - return pathlib.Path(path_to_this_file).resolve().parents[2] +class Paths: + def __init__(self, base_paths): + bootstrap_cfg = benchpark.config.configuration().bootstrap + self.benchpark_home = pathlib.Path(os.path.expanduser(bootstrap_cfg.location)) + self.global_ramble_path = self.benchpark_home / "ramble" + self.global_spack_path = self.benchpark_home / "spack" -benchpark_root = _source_location() -lib_path = benchpark_root / "lib" / "benchpark" -test_path = lib_path / "test" + self.base_paths = base_paths -benchpark_config = benchpark_root / "configure.yaml" + def __getattr__(self, name): + return getattr(self.base_paths, name) -if benchpark_config.exists(): - with benchpark_config.open("r") as f: - config = yaml.safe_load(f) - benchpark_home = config["bootstrap"]["location"] -else: - benchpark_home = "~/.benchpark" -benchpark_home = pathlib.Path(os.path.expanduser(benchpark_home)) - -global_ramble_path = benchpark_home / "ramble" -global_spack_path = benchpark_home / "spack" -hardware_descriptions = benchpark_root / "systems" / "all_hardware_descriptions" -checkout_versions = benchpark_root / "checkout-versions.yaml" -remote_urls = benchpark_root / "remote-urls.yaml" +paths = Paths(benchpark.base_paths.base_paths) +hardware_descriptions = paths.hardware_descriptions diff --git a/lib/benchpark/repo.py b/lib/benchpark/repo.py index ffff86b1f..72473e8b1 100644 --- a/lib/benchpark/repo.py +++ b/lib/benchpark/repo.py @@ -9,7 +9,7 @@ import sys from enum import Enum -import benchpark.paths +import benchpark.config # isort: off @@ -74,23 +74,8 @@ def override_ramble_hardcoded_globals(): ramble.language.language_base.namespaces = _old[2] -# Experiments -def _exprs(): - """Get the singleton RepoPath instance for Ramble. - - Create a RepoPath, add it to sys.meta_path, and return it. - - TODO: consider not making this a singleton. - """ - experiments_repo = benchpark.paths.benchpark_root / "experiments" - return _add_repo(experiments_repo, ObjectTypes.experiments) - - -def _add_repo(repo_dir, obj_type): - if repo_dir.exists(): - repo_dirs = [str(repo_dir)] - else: - raise ValueError(f"Repo dir does not exist: {repo_dir}") +def _add_repo(repo_dirs, obj_type): + repo_dirs = [str(x) for x in repo_dirs] with override_ramble_hardcoded_globals(): path = ramble.repository.RepoPath(*repo_dirs, object_type=obj_type) @@ -98,10 +83,16 @@ def _add_repo(repo_dir, obj_type): return path -# Systems +def _exprs(): + cfg = benchpark.config.configuration().repos + exp_repos = [cfg.resolve_path(x) for x in cfg.experiments] + return _add_repo(exp_repos, ObjectTypes.experiments) + + def _systems(): - systems_repo = benchpark.paths.benchpark_root / "systems" - return _add_repo(systems_repo, ObjectTypes.systems) + cfg = benchpark.config.configuration().repos + sys_repos = [cfg.resolve_path(x) for x in cfg.systems] + return _add_repo(sys_repos, ObjectTypes.systems) paths = { diff --git a/lib/benchpark/runtime.py b/lib/benchpark/runtime.py index 7866a4bb8..69b47fe36 100644 --- a/lib/benchpark/runtime.py +++ b/lib/benchpark/runtime.py @@ -12,8 +12,8 @@ import yaml -import benchpark.paths from benchpark.debug import debug_print +from benchpark.paths import paths @contextmanager @@ -75,7 +75,7 @@ def __init__(self, dest, upstream=None): ) # Read pinned versions of ramble and spack - with open(benchpark.paths.checkout_versions, "r") as yaml_file: + with open(paths.checkout_versions, "r") as yaml_file: data = yaml.safe_load(yaml_file)["versions"] self.ramble_commit, self.spack_commit, self.pkgs_commit = ( data["ramble"], @@ -84,7 +84,7 @@ def __init__(self, dest, upstream=None): ) # Read remote urls for ramble and spack - with open(benchpark.paths.remote_urls, "r") as yaml_file: + with open(paths.remote_urls, "r") as yaml_file: data = yaml.safe_load(yaml_file)["urls"] remote_ramble_url, remote_spack_url, remote_pkgs_url = ( data["ramble"], diff --git a/lib/benchpark/test/caliper.py b/lib/benchpark/test/caliper.py index 7aca10d05..bfab95663 100644 --- a/lib/benchpark/test/caliper.py +++ b/lib/benchpark/test/caliper.py @@ -10,8 +10,8 @@ from ramble.expander import Expander import benchpark.caliper -import benchpark.paths import benchpark.spec +from benchpark.paths import paths def get_caliper_vars_section(expr_spec): @@ -58,7 +58,7 @@ def test_caliper_modifier(monkeypatch): expr_vars_section = get_caliper_vars_section(expr_spec) # Append path to enable import of modifier and application - sys.path.append(str(benchpark.paths.benchpark_root)) + sys.path.append(str(paths.benchpark_root)) from modifiers.caliper.modifier import Caliper as CaliperModifier from repo.saxpy.application import Saxpy diff --git a/lib/benchpark/test/commands.py b/lib/benchpark/test/commands.py index 50b3f9674..93bcbdf11 100644 --- a/lib/benchpark/test/commands.py +++ b/lib/benchpark/test/commands.py @@ -5,14 +5,14 @@ import subprocess -import benchpark.paths +from benchpark.paths import paths def test_list(): for subcmd in ["experiments", "modifiers", "systems", "benchmarks"]: # Test with title (default behavior) result_with_title = subprocess.run( - [benchpark.paths.benchpark_root / "bin/benchpark", "list", subcmd], + [paths.benchpark_root / "bin/benchpark", "list", subcmd], check=True, capture_output=True, text=True, @@ -24,7 +24,7 @@ def test_list(): # Test without title (--no-title flag) result_no_title = subprocess.run( [ - benchpark.paths.benchpark_root / "bin/benchpark", + paths.benchpark_root / "bin/benchpark", "list", subcmd, "--no-title", @@ -46,7 +46,7 @@ def test_list(): # Check filtering check_cuda = subprocess.run( [ - benchpark.paths.benchpark_root / "bin/benchpark", + paths.benchpark_root / "bin/benchpark", "list", "experiments", "--experiment", @@ -64,7 +64,7 @@ def test_list(): def test_tags(): subprocess.run( - [benchpark.paths.benchpark_root / "bin/benchpark", "tags", "-a", "ad"], + [paths.benchpark_root / "bin/benchpark", "tags", "-a", "ad"], check=True, ) @@ -72,7 +72,7 @@ def test_tags(): def test_info(): text = subprocess.run( [ - benchpark.paths.benchpark_root / "bin/benchpark", + paths.benchpark_root / "bin/benchpark", "list", "systems", "--no-title", @@ -89,7 +89,7 @@ def test_info(): for r in result: subprocess.run( - [benchpark.paths.benchpark_root / "bin/benchpark", "info", "system", r], + [paths.benchpark_root / "bin/benchpark", "info", "system", r], check=True, capture_output=True, ) diff --git a/lib/benchpark/test/paths.py b/lib/benchpark/test/paths.py index 862124ca0..79b651374 100644 --- a/lib/benchpark/test/paths.py +++ b/lib/benchpark/test/paths.py @@ -5,9 +5,9 @@ import pathlib -import benchpark.paths +from benchpark.paths import paths def test_benchpark_root(pytestconfig): expected_path = pathlib.Path(pytestconfig.inipath).resolve().parent - assert benchpark.paths.benchpark_root == expected_path + assert paths.benchpark_root == expected_path diff --git a/lib/main.py b/lib/main.py index 7b37208be..95adde548 100755 --- a/lib/main.py +++ b/lib/main.py @@ -7,12 +7,16 @@ import argparse import inspect +import os +import pathlib import shlex import subprocess import sys import yaml +# Process --version/--help early and exit before doing any further imports +# (which can result in expensive cloning operations). __version__ = "0.1.0" if "-V" in sys.argv or "--version" in sys.argv: print(__version__) @@ -20,11 +24,9 @@ helpstr = """usage: main.py [-h] [-V] {tags,system,experiment,setup,unit-test,audit,mirror,info,show-build,list,bootstrap,analyze,aggregate,configure} ... Benchpark - options: -h, --help show this help message and exit -V, --version show version number and exit - Subcommands: {tags,system,experiment,setup,unit-test,audit,mirror,info,show-build,list,bootstrap,analyze,aggregate,configure} tags Tags in Benchpark experiments @@ -39,6 +41,7 @@ list List experiments, systems, benchmarks, and modifiers bootstrap Bootstrap benchpark or update an existing bootstrap analyze Perform pre-defined analysis on the performance data (caliper files) after 'ramble on' + redo Re-instantiate system and all associated experiments aggregate Aggregate multiple experiments (even across workspaces) into the same submission script configure Configure options relating to the Benchpark environment """ @@ -46,13 +49,36 @@ print(helpstr) exit() -import benchpark.paths # noqa: E402 -from benchpark.runtime import RuntimeResources # noqa: E402 +from benchpark.base_paths import base_paths # noqa: E402 + +# Set paths here that are important for configuration: after this block +# commands that use config can be imported and run +_found_cfg = False +for i, arg in enumerate(sys.argv): + if arg.startswith("-C") or arg.startswith("--config"): + _found_cfg = True + if "=" in arg: + val = arg.split("=")[1] + else: + val = sys.argv[i + 1] + base_paths.user_input_cfg = pathlib.Path(val) +if not _found_cfg: + base_paths.user_input_cfg = None + +base_paths.invocation_working_dir = pathlib.Path(os.getcwd()).absolute().resolve() -if sys.argv[1] == "configure": +# Later imports initiate bootstrapping, and the configure command may want +# to configure this behavior, so we handle it before doing remaining imports +if "configure" in sys.argv: import benchpark.cmd.configure # noqa: E402 parser = argparse.ArgumentParser(description="Benchpark") + parser.add_argument( + "-C", + "--config", + help="Use config related to system/experiment/workspace generation." + " Default is /config/", + ) subparsers = parser.add_subparsers(title="Subcommands", dest="subcommand") configure_parser = subparsers.add_parser( "configure", help="Configure options relating to the Benchpark environment" @@ -62,7 +88,10 @@ benchpark.cmd.configure.command(args) sys.exit(0) -bootstrapper = RuntimeResources(benchpark.paths.benchpark_home) # noqa +from benchpark.paths import paths # noqa: E402 +from benchpark.runtime import RuntimeResources # noqa: E402 + +bootstrapper = RuntimeResources(paths.benchpark_home) # noqa bootstrapper.bootstrap() # noqa import benchpark.cmd.aggregate # noqa: E402 @@ -95,6 +124,14 @@ def main(): parser.add_argument( "-V", "--version", action="store_true", help="show version number and exit" ) + # -C is actually processed above here, in the imports, but must + # be registered here because otherwise argparse will think there + # is an invalid option. + parser.add_argument( + "-C", + "--config", + help="use config related to system/experiment/workspace generation", + ) subparsers = parser.add_subparsers(title="Subcommands", dest="subcommand") @@ -143,7 +180,7 @@ def supports_unknown_args(command): def benchpark_get_tags(): - f = benchpark.paths.benchpark_root / "taxonomy.yaml" + f = paths.benchpark_root / "taxonomy.yaml" tags = [] with open(f, "r") as stream: @@ -335,8 +372,8 @@ def benchpark_tags_handler(args): """ Filter ramble tags by benchpark benchmarks """ - source_dir = benchpark.paths.benchpark_root - ramble_exe = benchpark.paths.benchpark_home / "ramble/bin/ramble" + source_dir = paths.benchpark_root + ramble_exe = paths.benchpark_home / "ramble/bin/ramble" subprocess.run([ramble_exe, "repo", "add", "--scope=site", f"{source_dir}/repo"]) benchmarks = benchpark_benchmarks() diff --git a/lib/scripts/diffExperimentBuilds.py b/lib/scripts/diffExperimentBuilds.py index 466387df1..83e898ced 100644 --- a/lib/scripts/diffExperimentBuilds.py +++ b/lib/scripts/diffExperimentBuilds.py @@ -4,9 +4,9 @@ import subprocess import sys -import benchpark.paths +from benchpark.paths import paths -sys.path.append(str(benchpark.paths.benchpark_home) + "/spack/lib/spack") +sys.path.append(str(paths.benchpark_home) + "/spack/lib/spack") import llnl.util.tty.color as color # noqa: E402 diff --git a/lib/scripts/diffExperimentSpecs.py b/lib/scripts/diffExperimentSpecs.py index db6235913..a867733f2 100644 --- a/lib/scripts/diffExperimentSpecs.py +++ b/lib/scripts/diffExperimentSpecs.py @@ -5,9 +5,9 @@ from diffSystems import compare_yaml -import benchpark.paths +from benchpark.paths import paths -sys.path.append(str(benchpark.paths.benchpark_home) + "/spack/lib/spack") +sys.path.append(str(paths.benchpark_home) + "/spack/lib/spack") import llnl.util.tty.color as color # noqa: E402 diff --git a/lib/scripts/diffPackageCommits.py b/lib/scripts/diffPackageCommits.py index 9d9a44b8a..376df0131 100644 --- a/lib/scripts/diffPackageCommits.py +++ b/lib/scripts/diffPackageCommits.py @@ -5,7 +5,7 @@ import subprocess import sys -import benchpark.paths +from benchpark.paths import paths EXIT_CODE = 0 @@ -39,7 +39,7 @@ def main(): global EXIT_CODE spack_dir = "spack/var/spack/repos/builtin/packages/" - benchpark_dir = str(benchpark.paths.benchpark_root) + "/repo/" + benchpark_dir = str(paths.benchpark_root) + "/repo/" # Get the list of packages to process if args.packages: diff --git a/lib/scripts/diffSystemSpecs.py b/lib/scripts/diffSystemSpecs.py index 9bc279f40..fa6c8c029 100644 --- a/lib/scripts/diffSystemSpecs.py +++ b/lib/scripts/diffSystemSpecs.py @@ -7,9 +7,9 @@ import yaml from deepdiff import DeepDiff -import benchpark.paths +from benchpark.paths import paths -sys.path.append(str(benchpark.paths.benchpark_home) + "/spack/lib/spack") +sys.path.append(str(paths.benchpark_home) + "/spack/lib/spack") import llnl.util.tty.color as color # noqa: E402