Custom repos in benchpark

.github/workflows/run.yml

@@ -222,10 +222,10 @@ jobs:
           pip install -r ./requirements.txt
       - name: Run
         run: |-
-          output=$(./bin/benchpark configure --bootstrap-location . && ./bin/benchpark bootstrap 2>&1)
+          output=$(./bin/benchpark configure --bootstrap-location ./bootstrap && ./bin/benchpark bootstrap 2>&1)
 
           # 1. Check the path is present
-          echo "$output" | grep -q "/home/runner/work/benchpark/benchpark/.benchpark/ramble" \
+          echo "$output" | grep -q "/home/runner/work/benchpark/benchpark/bootstrap/ramble" \
             || { echo "Expected path not found"; exit 1; }
 
           # 2. Check "Cloning Ramble" appears exactly once

config/bootstrap.yaml

@@ -0,0 +1,6 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+bootstrap:
+  location: ~/.benchpark/

config/repos.yaml

@@ -0,0 +1,9 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+repos:
+  experiments: [../experiments, ../experiments/test]
+  systems: [../systems]
+  applications: [../repo]
+  packages: [../repo]

docs/configuration.rst

@@ -0,0 +1,68 @@
+..
+    Copyright 2023 Lawrence Livermore National Security, LLC and other
+    Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+
+    SPDX-License-Identifier: Apache-2.0
+
+#######################
+ Configuring Benchpark
+#######################
+
+Benchpark offers several options to configure usage. This includes:
+
+1. Configuring the location of the bootstrap directory (default is ``~/.benchpark``).
+   This is useful to change from the default value if you do not have much space in your
+   home directory, as the bootstrap contains the ``spack``, ``spack-packages``, and
+   ``ramble`` repositories.
+
+1. Configuring which repositories benchpark uses for objects in the
+   ``experiments/systems/repos`` directories (``experiment.py``, ``system.py``,
+   ``package.py``, and ``application.py``).
+
+**********************************************
+ Configuring the Benchpark Bootstrap Location
+**********************************************
+
+Benchpark clones ``ramble``, ``spack``, and ``spack-packages`` into a centralized
+location (by default this is ``~/.benchpark``) to enable building and running
+benchmarks. Benchpark workspaces copy over these repositories from the centralized
+location to avoid downloading from the internet when initializing a Benchpark workspace.
+The bootstrap location can be configured in ``<benchpark_root>/config/bootstrap.yaml``
+or by running ``benchpark configure --bootstrap-location <location>``.
+
+***********************************************
+ Configuring Which Repositories Benchpark Uses
+***********************************************
+
+The ``<benchpark_root>/config/repos.yaml`` file is used to fully customize ``system``
+and ``experiment`` repositories used by Benchpark, ``application`` repositories used by
+Ramble, and package repositories used by Spack. For Spack and Ramble, the builtin
+repository is always used (in addition to whatever package repositories are specified).
+The order in which repositories appear in ``<benchpark_root>/config/repos.yaml`` will
+determine their precedence, so if you desire to use a custom repository, it should be
+listed before any other repositories.
+
+**********************************************************************
+ Automatically Generating Configurations with ``benchpark configure``
+**********************************************************************
+
+``benchpark configure`` is designed to create the ``yaml`` configurations for you. As of
+now, it can only generate bootstrap config (``bootstrap.yaml``). If no bootstrap config
+is detected in the chosen scope, this will auto-generate it. ``benchpark configure``
+cannot generate a ``repos.yaml`` file at the moment, this must be written manually.
+
+**********************
+ Configuration Scopes
+**********************
+
+Benchpark can pull configurations from one location, with the following priority
+(highest first):
+
+- As an argument to ``benchpark``: ``benchpark -C <dir>...``
+- If the CWD where you invoke the benchpark executable has ``benchpark-config``
+  directory
+- If ``<benchpark_root>/config`` is a directory; this must exist if the first two don't
+
+There is no mixing and matching between these tiers. If you are using ``-C``, then the
+specified directory must contain a ``bootstrap.yaml`` and a ``repos.yaml`` (you can copy
+from ``<benchpark_root>/config`` if you want the same config).

docs/generate-benchmark-list.py

@@ -8,7 +8,6 @@
 
 sys.path.append("../lib/")
 import benchpark.accounting  # noqa: E402
-import benchpark.paths  # noqa: E402
 
 
 def construct_tag_groups(tag_groups, tag_dicts, dictionary):

docs/index.rst

@@ -40,6 +40,7 @@
     run-experiment
     analyze-experiment
     benchpark-analyze
+    configuration
     modifiers
     set-of-experiments
     run-binary

experiments/test/repo.yaml

@@ -0,0 +1,7 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+repo:
+  namespace: test
+  subdirectory: ''

lib/benchpark/accounting.py

@@ -6,7 +6,7 @@
 import os
 from collections import defaultdict
 
-import benchpark.paths
+from benchpark.base_paths import base_paths
 
 PROGRAMMING_MODEL_CATEGORY = "programming_model"
 SCALING_CATEGORY = "scaling"
@@ -30,7 +30,7 @@
 
 
 def benchpark_experiments(exp_dict=EXP_DICT, exclude_variants=[]):
-    source_dir = benchpark.paths.benchpark_root
+    source_dir = base_paths.benchpark_root
     experiments = []
     experiments_dir = source_dir / "experiments"
 
@@ -60,7 +60,7 @@ def benchpark_experiments(exp_dict=EXP_DICT, exclude_variants=[]):
 
 def benchpark_modifiers():
     all_benchmark_modifiers = ["affinity", "allocation", "hwloc"]
-    source_dir = benchpark.paths.benchpark_root
+    source_dir = base_paths.benchpark_root
     experiments_dir = source_dir / "experiments"
     modifiers = []
     exclude = ["modifier_repo.yaml"]
@@ -88,7 +88,7 @@ def benchpark_modifiers():
 def benchpark_systems(programming_model=None):
     from benchpark.spec import SystemSpec
 
-    source_dir = benchpark.paths.benchpark_root
+    source_dir = base_paths.benchpark_root
     systems = []
     exclude = ["all_hardware_descriptions", "common", "repo.yaml"]
     for x in sorted(os.listdir(source_dir / "systems")):
@@ -128,7 +128,7 @@ def benchpark_systems(programming_model=None):
 
 
 def benchpark_benchmarks():
-    source_dir = benchpark.paths.benchpark_root
+    source_dir = base_paths.benchpark_root
     benchmarks = []
     experiments_dir = source_dir / "experiments"
     for x in sorted(os.listdir(experiments_dir)):

lib/benchpark/base_paths.py

@@ -0,0 +1,56 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import pathlib
+
+
+def _source_location() -> pathlib.Path:
+    """Return the location of the project source files directory."""
+    path_to_this_file = __file__
+    return pathlib.Path(path_to_this_file).resolve().parents[2]
+
+
+_unset = object()
+
+
+class BasePaths:
+    def __init__(self):
+        self.benchpark_root = _source_location()
+        self.lib_path = self.benchpark_root / "lib" / "benchpark"
+        self.test_path = self.lib_path / "test"
+        self.hardware_descriptions = (
+            self.benchpark_root / "systems" / "all_hardware_descriptions"
+        )
+        self.checkout_versions = self.benchpark_root / "checkout-versions.yaml"
+        self.remote_urls = self.benchpark_root / "remote-urls.yaml"
+        self.invocation_working_dir = None
+        self.user_input_cfg = _unset
+
+
+base_paths = BasePaths()
+
+
+def determine_config_dir():
+    """
+    Benchpark configs don't merge or override like Spack/Ramble. You
+    just point it at a directory and that's where all your config is.
+    """
+    if base_paths.user_input_cfg is _unset:
+        raise Exception("Internal error: config initialization")
+    elif base_paths.user_input_cfg:
+        if not base_paths.user_input_cfg.exists():
+            raise Exception(
+                f"Specific config dir does not exist: {base_paths.user_input_cfg}"
+            )
+        else:
+            return base_paths.user_input_cfg
+
+    possible_dirs = [
+        base_paths.invocation_working_dir / "benchpark-config",
+        base_paths.benchpark_root / "config",
+    ]
+    for pd in possible_dirs:
+        if pd.exists():
+            return pd

lib/benchpark/cmd/configure.py

@@ -8,7 +8,7 @@
 
 import yaml
 
-import benchpark.paths
+import benchpark.config
 
 
 def setup_parser(root_parser):
@@ -21,15 +21,14 @@ def setup_parser(root_parser):
 
 
 def command(args):
-    data = {}
-
-    if args.bootstrap_location:
-        loc = os.path.expandvars(os.path.expanduser(args.bootstrap_location))
-        bl = str(Path(loc).resolve()).rstrip("/") + "/.benchpark/"
-        data["bootstrap"] = {
-            "location": bl,
-        }
-
-    print(f"Writing configuration to {benchpark.paths.benchpark_config}")
-    with open(benchpark.paths.benchpark_config, "w") as yaml_file:
-        yaml.safe_dump(data, yaml_file)
+    bootstrap_cfg = benchpark.config.configuration().bootstrap
+
+    if args.bootstrap_location or not bootstrap_cfg.path.exists():
+        where = args.bootstrap_location or "~/.benchpark/"
+        loc = os.path.expandvars(os.path.expanduser(where))
+        bl = str(Path(loc).resolve()).rstrip("/")
+        data = {"bootstrap": {"location": bl}}
+
+        print(f"Writing configuration to {bootstrap_cfg.path}")
+        with open(bootstrap_cfg.path, "w") as yaml_file:
+            yaml.safe_dump(data, yaml_file)

lib/benchpark/cmd/info.py

@@ -5,7 +5,8 @@
 import llnl.util.tty.color as color
 import yaml
 
-import benchpark.paths
+import benchpark.spec
+from benchpark.paths import paths
 
 
 def indent():
@@ -71,7 +72,7 @@ def _handle_query(query):
         key, value = query.split("=", 1)
         exclude = {"all_hardware_descriptions", "repo.yaml", "generic-x86"}
         all_system_specs = []
-        for d in set(os.listdir(benchpark.paths.benchpark_root / "systems")) - exclude:
+        for d in set(os.listdir(paths.benchpark_root / "systems")) - exclude:
             all_system_specs.append(benchpark.spec.SystemSpec(d))
 
         matching_systems = []

lib/benchpark/cmd/mirror.py

@@ -12,7 +12,7 @@
 import shutil
 import tempfile
 
-import benchpark.paths
+from benchpark.paths import paths
 from benchpark.runtime import run_command, working_dir
 
 
@@ -115,7 +115,7 @@ def mirror_create(args):
         )
 
     cache_storage = os.path.join(dest, "pip-cache")
-    ramble_pip_reqs = os.path.join(benchpark.paths.benchpark_root, "requirements.txt")
+    ramble_pip_reqs = os.path.join(paths.benchpark_root, "requirements.txt")
     if not os.path.exists(cache_storage):
         run_command(f"pip download -r {ramble_pip_reqs} -d {cache_storage}")
 
@@ -159,7 +159,7 @@ def _ignore(path, dir_list):
     env_dir = os.path.dirname(find_one(ramble_workspace, "spack.yaml"))
     git_repo_dst = os.path.join(dest, "git-repos")
     repo_copy_script = os.path.join(
-        benchpark.paths.benchpark_root, "lib", "scripts", "env-collect-branch-tips.py"
+        paths.benchpark_root, "lib", "scripts", "env-collect-branch-tips.py"
     )
     out, err = run_command(
         f"spack -e {env_dir} python {repo_copy_script} {git_repo_dst}"
@@ -199,12 +199,12 @@ def _ignore(path, dir_list):
 """)
 
     modifiers_dest = os.path.join(dest, "modifiers")
-    modifiers_src = os.path.join(benchpark.paths.benchpark_root, "modifiers")
+    modifiers_src = os.path.join(paths.benchpark_root, "modifiers")
     if not os.path.exists(modifiers_dest):
         shutil.copytree(modifiers_src, modifiers_dest)
 
     bp_repo_dest = os.path.join(dest, "repo")
-    bp_repo_src = os.path.join(benchpark.paths.benchpark_root, "repo")
+    bp_repo_src = os.path.join(paths.benchpark_root, "repo")
     if not os.path.exists(bp_repo_dest):
         shutil.copytree(bp_repo_src, bp_repo_dest)
 

lib/benchpark/cmd/setup.py

@@ -12,8 +12,9 @@
 
 import ruamel.yaml as yaml
 
-import benchpark.paths
+import benchpark.config
 from benchpark.debug import debug_print
+from benchpark.paths import paths
 from benchpark.runtime import RuntimeResources
 
 
@@ -95,7 +96,7 @@ def _find(d, tag):
                     return result
 
     experiments_root = pathlib.Path(os.path.abspath(args.experiments_root))
-    source_dir = benchpark.paths.benchpark_root
+    source_dir = paths.benchpark_root
 
     experiment_src_dir = pathlib.Path(os.path.abspath(str(args.experiment)))
 
@@ -200,9 +201,11 @@ def include_fn(fname):
     run_script = experiments_root / ".latest-experiment.sh"
 
     per_workspace_setup = RuntimeResources(
-        experiments_root, upstream=RuntimeResources(benchpark.paths.benchpark_home)
+        experiments_root, upstream=RuntimeResources(paths.benchpark_home)
     )
 
+    repos_cfg = benchpark.config.configuration().repos
+
     pkg_str = ""
     if pkg_manager == "spack":
         spack_build_stage = experiments_root / "builds"
@@ -212,12 +215,18 @@ def include_fn(fname):
             site_repos = (
                 per_workspace_setup.spack_location / "etc" / "spack" / "repos.yaml"
             )
+            repos = {}
+            for repo_dir in reversed(repos_cfg.packages):
+                repo_dir = repos_cfg.resolve_path(repo_dir)
+                with open(repo_dir / "repo.yaml", "r") as f:
+                    repo_data = yaml.safe_load(f)
+                    namespace = repo_data["repo"]["namespace"]
+                    repos[namespace] = str(repo_dir)
+            repos["builtin"] = (
+                f"{per_workspace_setup.pkgs_location}/repos/spack_repo/builtin/"
+            )
             with open(site_repos, "w") as f:
-                f.write(f"""\
-repos::
-  benchpark: {source_dir}/repo
-  builtin: {per_workspace_setup.pkgs_location}/repos/spack_repo/builtin/
-""")
+                yaml.dump({"repos:": repos}, f, default_flow_style=False)
             spack(
                 f"config --scope=site add \"config:build_stage:['{spack_build_stage}']\""
             )
@@ -230,7 +239,9 @@ def include_fn(fname):
 
     ramble, first_time_ramble = per_workspace_setup.ramble_first_time_setup()
     if first_time_ramble:
-        ramble(f"repo add --scope=site {source_dir}/repo")
+        for repo_dir in reversed(repos_cfg.applications):
+            repo_dir = repos_cfg.resolve_path(repo_dir)
+            ramble(f"repo add --scope=site {repo_dir}")
         ramble('config --scope=site add "config:disable_progress_bar:true"')
         ramble(f"repo add -t modifiers --scope=site {source_dir}/modifiers")
         ramble("config --scope=site add \"config:spack:global:args:'-d'\"")