Better naming and values for MockEnv parameters + unit tests (#769)

* Use `mock_env_range`, `mock_env_seed`, and `mock_env_metrics` instead
of just `range`, `seed`, and `metrics` to avoid naming conflicts.
* Use integer `mocl_env_seed` values to distinguish between
deterministic behavior, random seed, and user-specified seed for
`MockEnv`.
* Update unit tests and configs to validate the seed behavior.
* Add new unit tests for different initializations of the tunable values

---------

Co-authored-by: Brian Kroth <[email protected]>

Author: motus

mlos_bench/mlos_bench/DEVNOTES.md

@@ -34,7 +34,7 @@ Each `Environment` config is a JSON5 file with the following structure:
         }
         // Environment constructor parameters
         // (specific to the Environment class being instantiated):
-        "seed": 42,
+        "mock_env_seed": 42,
         // ...
     }
 }

mlos_bench/mlos_bench/config/environments/mock/mock_env.jsonc

@@ -19,8 +19,8 @@
             "linux-hugepages-2048kB",
             "redis"
         ],
-        "seed": 42,            // Seed for the random noise generator. Omit to produce noise-free data.
-        "range": [60, 120],    // Range of the generated output values of the benchmark.
-        "metrics": ["score"]   // Names of fake benchmark metrics to generate. (Default is one metric, "score").
+        "mock_env_seed": 42,            // Seed for the random noise generator. Omit or set to 0 to produce noise-free data.
+        "mock_env_range": [60, 120],    // Range of the generated output values of the benchmark.
+        "mock_env_metrics": ["score"]   // Names of fake benchmark metrics to generate. (Default is one metric, "score").
     }
 }

mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json

@@ -19,12 +19,12 @@
                 },
                 {
                     "properties": {
-                        "seed": {
+                        "mock_env_seed": {
                             "type": "integer",
-                            "description": "Seed for the random number generator",
-                            "default": 0
+                            "description": "Seed for the random number generator. Set to -1 for deterministic behavior, 0 for default randomness.",
+                            "default": -1
                         },
-                        "range": {
+                        "mock_env_range": {
                             "type": "array",
                             "description": "Range of the random number generator",
                             "items": {
@@ -33,7 +33,7 @@
                             "minItems": 2,
                             "maxItems": 2
                         },
-                        "metrics": {
+                        "mock_env_metrics": {
                             "type": "array",
                             "description": "Names of fake benchmark metrics to be generate",
                             "items": {

mlos_bench/mlos_bench/environments/mock_env.py

@@ -17,7 +17,6 @@
 from mlos_bench.environments.status import Status
 from mlos_bench.environments.base_environment import Environment
 from mlos_bench.tunables import Tunable, TunableGroups, TunableValue
-from mlos_bench.util import nullable
 
 _LOG = logging.getLogger(__name__)
 
@@ -49,18 +48,19 @@ def __init__(self,
         global_config : dict
             Free-format dictionary of global parameters (e.g., security credentials)
             to be mixed in into the "const_args" section of the local config.
-            Optional arguments are `seed`, `range`, and `metrics`.
+            Optional arguments are `mock_env_seed`, `mock_env_range`, and `mock_env_metrics`.
+            Set `mock_env_seed` to -1 for deterministic behavior, 0 for default randomness.
         tunables : TunableGroups
             A collection of tunable parameters for *all* environments.
         service: Service
             An optional service object. Not used by this class.
         """
         super().__init__(name=name, config=config, global_config=global_config,
                          tunables=tunables, service=service)
-        seed = self.config.get("seed")
-        self._random = nullable(random.Random, seed)
-        self._range = self.config.get("range")
-        self._metrics = self.config.get("metrics", ["score"])
+        seed = int(self.config.get("mock_env_seed", -1))
+        self._random = random.Random(seed or None) if seed >= 0 else None
+        self._range = self.config.get("mock_env_range")
+        self._metrics = self.config.get("mock_env_metrics", ["score"])
         self._is_ready = True
 
     def run(self) -> Tuple[Status, datetime, Optional[Dict[str, TunableValue]]]:

mlos_bench/mlos_bench/optimizers/mock_optimizer.py

@@ -61,7 +61,7 @@ def suggest(self) -> TunableGroups:
         """
         tunables = super().suggest()
         if self._start_with_defaults:
-            _LOG.info("Use default values for the first trial")
+            _LOG.info("Use default tunable values")
             self._start_with_defaults = False
         else:
             for (tunable, _group) in tunables:

mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc

@@ -14,9 +14,10 @@
 
     "environment": "environments/mock/mock_env.jsonc",
 
-    "tunable_values": [
-        "tunable-values/tunable-values-example.jsonc"
-    ],
+    // Use default values for tunables unless otherwise specified
+    // "tunable_values": [
+    //     "tunable-values/tunable-values-example.jsonc"
+    // ],
 
     "globals": ["globals/global_test_config.jsonc"],
 

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-bad-metric-type.jsonc

@@ -2,7 +2,7 @@
     "name": "mock_env-full",
     "class": "mlos_bench.environments.mock_env.MockEnv",
     "config": {
-        "metrics": [
+        "mock_env_metrics": [
             {"bad": "metric type"}
         ]
     }

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-list-obj.jsonc

@@ -2,7 +2,7 @@
     "name": "partial mock env",
     "class": "mlos_bench.environments.MockEnv",
     "config": {
-        "seed": 42,
+        "mock_env_seed": 42,
         "const_args": {
             "foo": "bar",
             "int": 1,

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-nested-list.jsonc

@@ -2,7 +2,7 @@
     "name": "partial mock env",
     "class": "mlos_bench.environments.MockEnv",
     "config": {
-        "seed": 42,
+        "mock_env_seed": 42,
         "const_args": {
             "foo": "bar",
             "int": 1,

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-metric.jsonc

@@ -2,7 +2,7 @@
     "name": "mock_env-full",
     "class": "mlos_bench.environments.mock_env.MockEnv",
     "config": {
-        "metrics": [
+        "mock_env_metrics": [
             // needs at least one element
         ]
     }

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-name.jsonc

@@ -2,6 +2,6 @@
     // "name": "missing name is invalid",
     "class": "mlos_bench.environments.MockEnv",
     "config": {
-        "seed": 42
+        "mock_env_seed": 42
     }
 }

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/full/mock_env-full.jsonc

@@ -20,9 +20,9 @@
         "required_args": [
             "foo"
         ],
-        "range": [0, 1],
-        "seed": 42,
-        "metrics": [
+        "mock_env_range": [0, 1],
+        "mock_env_seed": 42,
+        "mock_env_metrics": [
             "latency",
             "cost"
         ]

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/partial/mock_env-const-args.jsonc

@@ -2,7 +2,7 @@
     "name": "partial mock env",
     "class": "mlos_bench.environments.MockEnv",
     "config": {
-        "seed": 42,
+        "mock_env_seed": 42,
         "const_args": {
             "foo": "bar",
             "int": 1,

mlos_bench/mlos_bench/tests/config/tunable-values/tunable-values-example.jsonc

@@ -2,8 +2,10 @@
 {
     "$schema": "https://raw.githubusercontent.com/microsoft/MLOS/main/mlos_bench/mlos_bench/config/schemas/tunables/tunable-values-schema.json",
 
-    "sched_migration_cost_ns": 500000,
-    "sched_latency_ns": 12000000,
+    // Values that are different from the defaults
+    // in order to test --tunable-values handling in OneShotOptimizer.
+    "sched_migration_cost_ns": 400000,
+    "sched_latency_ns": 10000000,
     "sched_child_runs_first": "0",
     "sched_tunable_scaling": "1"
 }

mlos_bench/mlos_bench/tests/conftest.py

@@ -40,9 +40,9 @@ def mock_env(tunable_groups: TunableGroups) -> MockEnv:
         name="Test Env",
         config={
             "tunable_params": ["provision", "boot", "kernel"],
-            "seed": SEED,
-            "range": [60, 120],
-            "metrics": ["score"],
+            "mock_env_seed": SEED,
+            "mock_env_range": [60, 120],
+            "mock_env_metrics": ["score"],
         },
         tunables=tunable_groups
     )
@@ -57,8 +57,9 @@ def mock_env_no_noise(tunable_groups: TunableGroups) -> MockEnv:
         name="Test Env No Noise",
         config={
             "tunable_params": ["provision", "boot", "kernel"],
-            "range": [60, 120],
-            "metrics": ["score", "other_score"],
+            "mock_env_seed": -1,
+            "mock_env_range": [60, 120],
+            "mock_env_metrics": ["score", "other_score"],
         },
         tunables=tunable_groups
     )

mlos_bench/mlos_bench/tests/environments/composite_env_test.py

@@ -41,8 +41,8 @@ def composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
                             "EnvId": 1,
                         },
                         "required_args": ["vmName", "someConst", "global_param"],
-                        "range": [60, 120],
-                        "metrics": ["score"],
+                        "mock_env_range": [60, 120],
+                        "mock_env_metrics": ["score"],
                     }
                 },
                 {
@@ -56,8 +56,8 @@ def composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
                             "global_param": "local"
                         },
                         "required_args": ["vmName"],
-                        "range": [60, 120],
-                        "metrics": ["score"],
+                        "mock_env_range": [60, 120],
+                        "mock_env_metrics": ["score"],
                     }
                 },
                 {
@@ -70,8 +70,8 @@ def composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
                             "EnvId": 3,
                         },
                         "required_args": ["vmName", "vm_server_name", "vm_client_name"],
-                        "range": [60, 120],
-                        "metrics": ["score"],
+                        "mock_env_range": [60, 120],
+                        "mock_env_metrics": ["score"],
                     }
                 }
             ]
@@ -193,8 +193,8 @@ def nested_composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
                                         "vm_server_name",
                                         "global_param"
                                     ],
-                                    "range": [60, 120],
-                                    "metrics": ["score"],
+                                    "mock_env_range": [60, 120],
+                                    "mock_env_metrics": ["score"],
                                 }
                             },
                             # ...
@@ -218,8 +218,8 @@ def nested_composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
                                 "config": {
                                     "tunable_params": ["boot"],
                                     "required_args": ["vmName", "EnvId", "vm_client_name"],
-                                    "range": [60, 120],
-                                    "metrics": ["score"],
+                                    "mock_env_range": [60, 120],
+                                    "mock_env_metrics": ["score"],
                                 }
                             },
                             # ...

mlos_bench/mlos_bench/tests/launcher_in_process_test.py

@@ -17,11 +17,14 @@
     ("argv", "expected_score"), [
         ([
             "--config", "mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc",
-        ], 65.6742),
+            "--trial_config_repeat_count", "5",
+            "--mock_env_seed", "-1",  # Deterministic Mock Environment.
+        ], 67.40329),
         ([
             "--config", "mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc",
             "--trial_config_repeat_count", "3",
             "--max_suggestions", "3",
+            "--mock_env_seed", "42",  # Noisy Mock Environment.
         ], 64.53897),
     ]
 )

mlos_bench/mlos_bench/tests/launcher_run_test.py

@@ -52,7 +52,9 @@ def _launch_main_app(root_path: str, local_exec_service: LocalExecService,
         # temp_dir = '/tmp'
         log_path = path_join(temp_dir, "mock-test.log")
         (return_code, _stdout, _stderr) = local_exec_service.local_exec(
-            [f"./mlos_bench/mlos_bench/run.py {cli_config} --log_file '{log_path}'"],
+            ["./mlos_bench/mlos_bench/run.py" +
+             " --config_path ./mlos_bench/mlos_bench/tests/config/" +
+             f" {cli_config} --log_file '{log_path}'"],
             cwd=root_path)
         assert return_code == 0
 
@@ -74,14 +76,35 @@ def _launch_main_app(root_path: str, local_exec_service: LocalExecService,
 def test_launch_main_app_bench(root_path: str, local_exec_service: LocalExecService) -> None:
     """
     Run mlos_bench command-line application with mock benchmark config
-    and check the results in the log.
+    and default tunable values and check the results in the log.
+    """
+    _launch_main_app(
+        root_path, local_exec_service,
+        " --config cli/mock-bench.jsonc" +
+        " --trial_config_repeat_count 5" +
+        " --mock_env_seed -1",  # Deterministic Mock Environment.
+        [
+            f"^{_RE_DATE} run\\.py:\\d+ " +
+            r"_main INFO Final score: \{'score': 67\.40\d+\}\s*$",
+        ]
+    )
+
+
+def test_launch_main_app_bench_values(
+        root_path: str, local_exec_service: LocalExecService) -> None:
+    """
+    Run mlos_bench command-line application with mock benchmark config
+    and user-specified tunable values and check the results in the log.
     """
     _launch_main_app(
         root_path, local_exec_service,
-        "--config mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc",
+        " --config cli/mock-bench.jsonc" +
+        " --tunable_values tunable-values/tunable-values-example.jsonc" +
+        " --trial_config_repeat_count 5" +
+        " --mock_env_seed -1",  # Deterministic Mock Environment.
         [
             f"^{_RE_DATE} run\\.py:\\d+ " +
-            r"_main INFO Final score: \{'score': 65\.67\d+\}\s*$",
+            r"_main INFO Final score: \{'score': 67\.11\d+\}\s*$",
         ]
     )
 
@@ -93,7 +116,10 @@ def test_launch_main_app_opt(root_path: str, local_exec_service: LocalExecServic
     """
     _launch_main_app(
         root_path, local_exec_service,
-        "--config mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc --trial_config_repeat_count 3 --max_suggestions 3",
+        "--config cli/mock-opt.jsonc" +
+        " --trial_config_repeat_count 3" +
+        " --max_suggestions 3" +
+        " --mock_env_seed 42",  # Noisy Mock Environment.
         [
             # Iteration 1: Expect first value to be the baseline
             f"^{_RE_DATE} mlos_core_optimizer\\.py:\\d+ " +