teuthology/suite: wire in the graph representation

Reviewer's note: we will probably want to make the matrix representation
a valid fallback using a command line switch. This is also important
when doing reruns with subsets generated via the matrix representation.

Signed-off-by: Patrick Donnelly <[email protected]>

Author: Patrick Donnelly

teuthology/suite/merge.py

@@ -132,23 +132,17 @@ def config_merge(configs, suite_name=None, **kwargs):
 
         yaml_complete_obj = copy.deepcopy(base_config.to_dict())
         deep_merge(yaml_complete_obj, dict(TEUTHOLOGY_TEMPLATE))
-        for path in paths:
-            if path not in yaml_cache:
-                with open(path) as f:
-                    txt = f.read()
-                    yaml_cache[path] = (txt, yaml.safe_load(txt))
-
-            yaml_fragment_txt, yaml_fragment_obj = yaml_cache[path]
+        for (path, yaml_fragment_obj) in paths:
             if yaml_fragment_obj is None:
                 continue
             yaml_fragment_obj = copy.deepcopy(yaml_fragment_obj)
             premerge = yaml_fragment_obj.get('teuthology', {}).pop('premerge', '')
             if premerge:
                 log.debug("premerge script running:\n%s", premerge)
                 env, script = new_script(premerge, log, deep_merge, yaml.safe_load)
-                env['base_frag_paths'] = [strip_fragment_path(x) for x in paths]
+                env['base_frag_paths'] = [strip_fragment_path(path) for (path, yaml) in paths]
                 env['description'] = desc
-                env['frag_paths'] = paths
+                env['frag_paths'] = [path for (path, yaml) in paths]
                 env['suite_name'] = suite_name
                 env['yaml'] = yaml_complete_obj
                 env['yaml_fragment'] = yaml_fragment_obj
@@ -164,9 +158,9 @@ def config_merge(configs, suite_name=None, **kwargs):
         postmerge = "\n".join(postmerge)
         log.debug("postmerge script running:\n%s", postmerge)
         env, script = new_script(postmerge, log, deep_merge, yaml.safe_load)
-        env['base_frag_paths'] = [strip_fragment_path(x) for x in paths]
+        env['base_frag_paths'] = [strip_fragment_path(path) for (path, yaml) in paths]
         env['description'] = desc
-        env['frag_paths'] = paths
+        env['frag_paths'] = [path for (path, yaml) in paths]
         env['suite_name'] = suite_name
         env['yaml'] = yaml_complete_obj
         for k,v in kwargs.items():

teuthology/suite/run.py

@@ -21,7 +21,7 @@
 
 from teuthology.suite import util
 from teuthology.suite.merge import config_merge
-from teuthology.suite.build_matrix import build_matrix
+from teuthology.suite.build_graph import build_graph
 from teuthology.suite.placeholder import substitute_placeholders, dict_templ
 from teuthology.util.time import parse_offset, parse_timestamp, TIMESTAMP_FMT
 
@@ -627,10 +627,12 @@ def schedule_suite(self):
         if self.args.dry_run:
             log.debug("Base job config:\n%s" % self.base_config)
 
-        configs = build_matrix(suite_path,
-                               subset=self.args.subset,
-                               no_nested_subset=self.args.no_nested_subset,
-                               seed=self.args.seed)
+        configs = build_graph(suite_path,
+                              subset=self.args.subset,
+                              no_nested_subset=self.args.no_nested_subset,
+                              seed=self.args.seed,
+                              suite_repo_path=self.suite_repo_path,
+                              config=config)
         generated = len(configs)
         log.info(f'Suite {suite_name} in {suite_path} generated {generated} jobs (not yet filtered or merged)')
         configs = list(config_merge(configs,