Add entry_point plugin support for user defined HELM run_specs

.github/workflows/tests.yml

@@ -30,6 +30,7 @@ jobs:
       run: |-
         python -m pip install pip uv -U
         python -m uv pip install -r pyproject.toml --extra tests
+        python -m uv pip install -r requirements/runtime.txt
     - name: Build sdist
       shell: bash
       run: |-
@@ -205,6 +206,7 @@ jobs:
         echo "UV_RESOLUTION=$UV_RESOLUTION"
         echo "MOD_VERSION=$MOD_VERSION"
         python -m uv pip install --prerelease=allow "aiq-magnet[$INSTALL_EXTRAS]==$MOD_VERSION" -f wheelhouse
+        python -m uv pip install -r requirements/runtime.txt
         echo "Install finished."
     - name: Test wheel ${{ matrix.install-extras }}
       shell: bash

magnet/helm_outputs.py

@@ -85,7 +85,6 @@ class HelmOutputs(ub.NiceRepr):
             │           └─╼  ...
             ├─╼ scenario_instances
             └─╼ runs: .size=4.00 KB,.files=1,json.size=2.64 MB,json.files=88,tex.size=13.11 KB,tex.files=42
-                ├─╼ latest -> my-suite: size=4.00 KB
                 └─╼ my-suite: json.size=2.64 MB,json.files=88,tex.size=13.11 KB,tex.files=42
                     ├─╼ runs_to_run_suites.json: size=0.36 KB
         ...
@@ -262,6 +261,7 @@ def suites(self, pattern='*') -> list[HelmSuite]:
     def _suite_dirs(self, pattern='*'):
         # not robust to extra directories being written.  is there a way to
         # determine that these directories are actually suites?
+        # TODO: no longer need to handle latest.
         return [p for p in sorted((self.root_dir / 'runs').glob(pattern)) if p.is_dir() and p.name != 'latest']
 
     def list_suites(self):
@@ -471,7 +471,7 @@ def _coerce_from_patterned_paths(cls, input):
             >>> cases = [
             >>>     {'num_expect': 1, 'input': root_dir / 'runs' / 'my-suite'},
             >>>     {'num_expect': 1, 'input': root_dir / 'runs' / '*suite*'},
-            >>>     {'num_expect': 2, 'input': root_dir / 'runs' / '*'},
+            >>>     {'num_expect': 1, 'input': root_dir / 'runs' / '*'},  # was 2 for helm 0.5.11, but symlinks were removed
             >>> ]
             >>> for case in cases:
             >>>     input = case['input']
@@ -502,6 +502,8 @@ def _coerce_from_patterned_paths(cls, input):
             # We could just check for is_dir and name != latest like
             # outputs does instead of using _is_likely_a_suite_path
             # unsure what the right answer is.
+            # NOTE: latest was removed in
+            # https://github.com/stanford-crfm/helm/pull/3984
             path = ub.Path(path)
             if HelmSuite._is_likely_a_suite_path(path):
                 suite_paths.append(path)
@@ -646,7 +648,7 @@ def _coerce_from_patterned_paths(cls, input):
             >>>     {'num_expect': 2, 'input': root_dir / 'runs' / 'my-suite/*subject=philosophy*'},
             >>>     {'num_expect': 2, 'input': root_dir / 'runs' / 'my-suite/*subject=anatomy*'},
             >>>     {'num_expect': 4, 'input': root_dir / 'runs' / 'my-suite/*:*'},
-            >>>     {'num_expect': 8, 'input': root_dir / 'runs' / '*/*:*'},
+            >>>     {'num_expect': 4, 'input': root_dir / 'runs' / '*/*:*'},
             >>> ]
             >>> for case in cases:
             >>>     input = case['input']

pyproject.toml

@@ -76,6 +76,9 @@ docs = [
 magnet = "magnet.__main__:main"
 magnet-helm-run = "magnet.backends.helm.magnet_run_helm:main"
 
+[project.entry-points.helm]
+run_specs = "magnet.backends.helm.run_specs"
+
 [tool.pytest.ini_options]
 addopts = "--xdoctest --xdoctest-style=google --ignore-glob=docs"
 norecursedirs = ".git build __pycache__ dev docs dockerfiles"

requirements/runtime.txt

@@ -0,0 +1 @@
+crfm-helm @ git+https://github.com/AIQ-Kitware/helm@kitware-main