feat: History based loss metric (foundation-model-stack#156)

* feat: History based loss metric

Signed-off-by: Padmanabha V Seshadri <[email protected]>

* feat: History based loss metric

Signed-off-by: Padmanabha V Seshadri <[email protected]>

* fix: corrected the keys and rules

Signed-off-by: Padmanabha V Seshadri <[email protected]>

* fix: Formatting issues in the code

Signed-off-by: Padmanabha V Seshadri <[email protected]>

* feat: Added patience configuration

Signed-off-by: Padmanabha V Seshadri <[email protected]>

* fix: Added info msgs to patience

Signed-off-by: Padmanabha V Seshadri <[email protected]>

* fix: Formatting issues resolved

Signed-off-by: Padmanabha V Seshadri <[email protected]>

* feat: Test cases for history based metrics

Signed-off-by: Padmanabha V Seshadri <[email protected]>

* fix: Format issues resolved

Signed-off-by: Padmanabha V Seshadri <[email protected]>

* fix: Addressed review comments on patience

Signed-off-by: Padmanabha V Seshadri <[email protected]>

* feat: patience upgraded

Signed-off-by: Padmanabha V Seshadri <[email protected]>

* fix: Handle repeated entries

Signed-off-by: Padmanabha V Seshadri <[email protected]>

* fix: Duplicate logs and other comments addressed

Signed-off-by: Padmanabha V Seshadri <[email protected]>

* fix: ADR changed to reflect convension change

Signed-off-by: Padmanabha V Seshadri <[email protected]>

* feat: Added examples for eval and training loss thresholds

Signed-off-by: Padmanabha V Seshadri <[email protected]>

* feat: Added examples for eval and training loss thresholds

Signed-off-by: Padmanabha V Seshadri <[email protected]>

---------

Signed-off-by: Padmanabha V Seshadri <[email protected]>

architecture_records/001-trainer-controller-framework.md

@@ -98,7 +98,7 @@ controller-metrics:
   - name: loss
     class: Loss
 controllers:
-  - name: loss-controller
+  - name: loss_controller
     triggers:
       - on_log
     rule: "loss < 1.0"
@@ -107,9 +107,8 @@ controllers:
 ```
 
 We follow the below naming convention for the above trainer controller configuration:
-1. `-` could be used in the case of key names, and name of the metric, operation or controller. This is usually to break multiple words of a name phrase.
 1. Python convention for [class name](https://visualgit.readthedocs.io/en/latest/pages/naming_convention.html#classes).
-1. `_` are used for events and control actions.
+1. `_` should be used between words in keys, values, events and control actions.
 
 For defining custom handler classes, we have an interface defined as an abstract class as shown below, with two abstract methods, namely: `validate()` to define the validation conditions, and `compute()` to compute the metric. The `compute()` returns an `Any` type. While it could be any value, developers should keep in mind that it should be only key-value pairs that are used in the rule(s) defined in the configuration.
 

examples/trainercontroller_configs/epoch-level-eval-loss-below-threshold.yaml

@@ -0,0 +1,12 @@
+controller_metrics:
+  - name: trainer_state
+    class: TrainingState
+  - name: evalmetric
+    class: EvalMetrics
+controllers:
+  - name: epoch_level_eval_loss_below_threshold
+    triggers:
+      - on_epoch_end
+    rule: evalmetric['eval_loss'] < 2.25 and trainer_state["epoch"] > 2
+    operations:
+      - hfcontrols.should_training_stop

examples/trainercontroller_configs/epoch-level-eval-loss-patience.yaml

@@ -0,0 +1,14 @@
+controller_metrics:
+  - name: eval_loss_window
+    class: HistoryBasedMetric
+    arguments:
+      window_size: 2
+controllers:
+  - name: epoch_level_eval_loss_patience
+    triggers:
+      - on_epoch_end
+    rule: len(eval_loss_window["metrics"]) > 0 and eval_loss_window["metrics"]["eval_loss"][-1] > 2
+    patience:
+      patience_threshold: 2
+    operations:
+      - hfcontrols.should_training_stop

examples/trainercontroller_configs/epoch-level-eval-loss.yaml

@@ -0,0 +1,14 @@
+controller_metrics:
+  - name: trainer_state
+    class: TrainingState
+  - name: eval_loss_window
+    class: HistoryBasedMetric
+    arguments:
+      window_size: 1
+controllers:
+  - name: epoch_level_eval_loss
+    triggers:
+      - on_epoch_end
+    rule: len(eval_loss_window["metrics"]) > 0 and eval_loss_window["metrics"]["eval_loss"][-1] > 2.2 and trainer_state["epoch"] > 3
+    operations:
+      - hfcontrols.should_training_stop

examples/trainercontroller_configs/epoch-level-training-loss-below-threshold.yaml

@@ -0,0 +1,12 @@
+controller_metrics:
+  - name: training_loss_window
+    class: HistoryBasedMetric
+    arguments:
+      window_size: 1
+controllers:
+  - name: epoch_level_stop_on_training_loss_below_threshold
+    triggers:
+      - on_log
+    rule: len(training_loss_window["training_loss"]["loss"]) == training_loss_window["window_size"] and training_loss_window["training_loss"]["loss"][0] < 2.2 and training_loss_window["training_loss"]["epoch"][0] > 2
+    operations:
+      - hfcontrols.should_training_stop

examples/trainercontroller_configs/epoch-level-training-loss.yaml

@@ -0,0 +1,14 @@
+controller_metrics:
+  - name: trainer_state
+    class: TrainingState
+  - name: training_loss_window
+    class: HistoryBasedMetric
+    arguments:
+      window_size: 1
+controllers:
+  - name: epoch_level_training_loss
+    triggers:
+      - on_epoch_end
+    rule: training_loss_window["training_loss"]["loss"][-1] > 2 and trainer_state["epoch"] > 3
+    operations:
+      - hfcontrols.should_training_stop

examples/trainercontroller_configs/loss.yaml

@@ -1,8 +1,8 @@
-controller-metrics:
+controller_metrics:
   - name: loss
     class: Loss
 controllers:
-  - name: loss-controller
+  - name: loss_controller
     triggers:
       - on_log
     rule: loss < 1.0

examples/trainercontroller_configs/non-decreasing-training-loss.yaml

@@ -0,0 +1,12 @@
+controller_metrics:
+  - name: training_loss_window
+    class: HistoryBasedMetric
+    arguments:
+      window_size: 5
+controllers:
+  - name: stop_on_training_loss_not_decreasing
+    triggers:
+      - on_log
+    rule: training_loss_window["training_loss"]["loss"][0] < training_loss_window["training_loss"]["loss"][-1] and len(training_loss_window["training_loss"]["loss"]) == training_loss_window["window_size"]
+    operations:
+      - hfcontrols.should_training_stop

examples/trainercontroller_configs/thresholded-training-loss.yaml

@@ -0,0 +1,12 @@
+controller_metrics:
+  - name: training_loss_window
+    class: HistoryBasedMetric
+    arguments:
+      window_size: 1
+controllers:
+  - name: stop_on_training_loss_not_decreasing
+    triggers:
+      - on_log
+    rule: training_loss_window["training_loss"]["loss"][-1] > 2.2
+    operations:
+      - hfcontrols.should_training_stop

tests/data/trainercontroller/__init__.py

@@ -62,3 +62,18 @@
 TRAINER_CONFIG_TEST_CUSTOM_OPERATION_INVALID_ACTION_YAML = os.path.join(
     _DATA_DIR, "loss_custom_operation_invalid_action.yaml"
 )
+TRAINER_CONFIG_TEST_EPOCH_LEVEL_EVAL_LOSS_PATIENCE_YAML = os.path.join(
+    _DATA_DIR, "epoch-level-eval-loss-patience.yaml"
+)
+TRAINER_CONFIG_TEST_EPOCH_LEVEL_EVAL_LOSS_YAML = os.path.join(
+    _DATA_DIR, "epoch-level-eval-loss.yaml"
+)
+TRAINER_CONFIG_TEST_EPOCH_LEVEL_TRAINING_LOSS_YAML = os.path.join(
+    _DATA_DIR, "epoch-level-training-loss.yaml"
+)
+TRAINER_CONFIG_TEST_NON_DECREASING_TRAINING_LOSS_YAML = os.path.join(
+    _DATA_DIR, "non-decreasing-training-loss.yaml"
+)
+TRAINER_CONFIG_TEST_THRESHOLDED_TRAINING_LOSS_YAML = os.path.join(
+    _DATA_DIR, "thresholded-training-loss.yaml"
+)

tests/data/trainercontroller/epoch-level-eval-loss-patience.yaml

@@ -0,0 +1,14 @@
+controller_metrics:
+  - name: eval_loss_window
+    class: HistoryBasedMetric
+    arguments:
+      window_size: 2
+controllers:
+  - name: epoch_level_eval_loss_patience
+    triggers:
+      - on_epoch_end
+    rule: len(eval_loss_window["metrics"]["eval_loss"]) > 0 and eval_loss_window["metrics"]["eval_loss"][-1] > 2
+    patience:
+      patience_threshold: 2
+    operations:
+      - hfcontrols.should_training_stop

tests/data/trainercontroller/epoch-level-eval-loss.yaml

@@ -0,0 +1,14 @@
+controller_metrics:
+  - name: trainer_state
+    class: TrainingState
+  - name: eval_loss_window
+    class: HistoryBasedMetric
+    arguments:
+      window_size: 1
+controllers:
+  - name: epoch_level_eval_loss
+    triggers:
+      - on_epoch_end
+    rule: len(eval_loss_window["metrics"]["eval_loss"]) > 0 and eval_loss_window["metrics"]["eval_loss"][-1] > 2 and trainer_state["epoch"] > 0.1
+    operations:
+      - hfcontrols.should_training_stop

tests/data/trainercontroller/epoch-level-training-loss.yaml

@@ -0,0 +1,14 @@
+controller_metrics:
+  - name: trainer_state
+    class: TrainingState
+  - name: training_loss_window
+    class: HistoryBasedMetric
+    arguments:
+      window_size: 1
+controllers:
+  - name: epoch_level_training_loss
+    triggers:
+      - on_epoch_end
+    rule: training_loss_window["training_loss"]["loss"][-1] < 1 and trainer_state["epoch"] >= 0.5
+    operations:
+      - hfcontrols.should_training_stop

tests/data/trainercontroller/exposed_metrics.yaml

@@ -1,10 +1,10 @@
-controller-metrics:
+controller_metrics:
   - name: evalmetric
     class: EvalMetrics
     arguments:
-      source-event: on_evaluate
+      source_event: on_evaluate
 controllers:
-  - name: loss-controller
+  - name: loss_controller
     triggers:
       - on_evaluate
     rule: evalmetric['eval_loss'] < 2.5

tests/data/trainercontroller/incorrect_source_event_exposed_metrics.yaml

@@ -1,10 +1,10 @@
-controller-metrics:
+controller_metrics:
   - name: evalmetric
     class: EvalMetrics
     arguments:
-      source-event: on_incorrect_event
+      source_event: on_incorrect_event
 controllers:
-  - name: loss-controller
+  - name: loss_controller
     triggers:
       - on_evaluate
     rule: evalmetric['eval_loss'] < 2.5

tests/data/trainercontroller/loss_custom_metric.yaml

@@ -1,8 +1,8 @@
-controller-metrics:
+controller_metrics:
   - name: testflag
     class: CustomMetric
 controllers:
-  - name: loss-controller-custom-metric
+  - name: loss_controller_custom-metric
     triggers:
       - on_log
     rule: testflag == True

tests/data/trainercontroller/loss_custom_operation.yaml

@@ -1,13 +1,13 @@
-controller-metrics:
+controller_metrics:
   - name: loss
     class: Loss
 operations:
-  - name: customoperation
+  - name: custom_operation
     class: CustomOperation
 controllers:
-  - name: loss-controller-custom-operation
+  - name: loss_controller_custom_operation
     triggers:
       - on_log
     rule: loss < 1.0
     operations:
-      - customoperation.should_perform_action_xyz
+      - custom_operation.should_perform_action_xyz

tests/data/trainercontroller/loss_custom_operation_invalid_action.yaml

@@ -1,13 +1,13 @@
-controller-metrics:
+controller_metrics:
   - name: loss
     class: Loss
 operations:
-  - name: customoperation
+  - name: custom_operation
     class: CustomOperationInvalidAction
 controllers:
-  - name: loss-controller-custom-operation-invalid-action
+  - name: loss_controller_custom_operation_invalid_action
     triggers:
       - on_log
     rule: loss < 1.0
     operations:
-      - customoperation.should_
+      - custom_operation.should_

tests/data/trainercontroller/loss_invalid_metric.yaml

@@ -1,8 +1,8 @@
-controller-metrics:
+controller_metrics:
   - name: loss
     class: MissingMetricClass
 controllers:
-  - name: loss-controller-invalid-metric
+  - name: loss_controller_invalid_metric
     triggers:
       - on_log
     rule: loss < 1.0

tests/data/trainercontroller/loss_invalid_operation.yaml

@@ -1,8 +1,8 @@
-controller-metrics:
+controller_metrics:
   - name: loss
     class: Loss
 controllers:
-  - name: loss-controller-invalid-operation
+  - name: loss_controller_invalid_operation
     triggers:
       - on_log
     rule: loss < 1.0

tests/data/trainercontroller/loss_invalid_operation_action.yaml

@@ -1,8 +1,8 @@
-controller-metrics:
+controller_metrics:
   - name: loss
     class: Loss
 controllers:
-  - name: loss-controller-invalid-operation-action
+  - name: loss_controller_invalid_operation_action
     triggers:
       - on_log
     rule: loss < 1.0

tests/data/trainercontroller/loss_invalid_trigger.yaml

@@ -1,8 +1,8 @@
-controller-metrics:
+controller_metrics:
   - name: loss
     class: Loss
 controllers:
-  - name: loss-controller-invalid-trigger
+  - name: loss_controller_invalid_trigger
     triggers:
       - log_it_all_incorrect_trigger_name
     rule: loss < 1.0

tests/data/trainercontroller/loss_on_threshold.yaml

@@ -1,8 +1,8 @@
-controller-metrics:
+controller_metrics:
   - name: loss
     class: Loss
 controllers:
-  - name: loss-controller
+  - name: loss_controller
     triggers:
       - on_log
     rule: loss < 1.0

tests/data/trainercontroller/loss_on_threshold_with_trainer_state.yaml

@@ -1,10 +1,10 @@
-controller-metrics:
+controller_metrics:
   - name: state
     class: TrainingState
   - name: loss
     class: Loss
 controllers:
-  - name: loss-controller
+  - name: loss_controller
     triggers:
       - on_log
     rule: loss < 2 and state["epoch"] >= 0.5

tests/data/trainercontroller/loss_unavailable_metric.yaml

@@ -1,8 +1,8 @@
-controller-metrics:
+controller_metrics:
   - name: loss
     class: Loss
 controllers:
-  - name: loss-controller-unavailable-metric
+  - name: loss_controller_unavailable_metric
     triggers:
       - on_step_end
     rule: loss < 1.0

tests/data/trainercontroller/loss_with_invalid_type_rule.yaml

@@ -1,8 +1,8 @@
-controller-metrics:
+controller_metrics:
   - name: loss
     class: Loss
 controllers:
-  - name: loss-controller-wrong-os-rule
+  - name: loss_controller_wrong_os_rule
     triggers:
       - on_log
     rule: "2+2"

tests/data/trainercontroller/loss_with_malicious_input_rule.yaml

@@ -1,8 +1,8 @@
-controller-metrics:
+controller_metrics:
   - name: loss
     class: Loss
 controllers:
-  - name: loss-controller-wrong-input-rule
+  - name: loss_controller_wrong_input_rule
     triggers:
       - on_log
     rule: input('Please enter your password:')

tests/data/trainercontroller/loss_with_malicious_os_rule.yaml

@@ -1,8 +1,8 @@
-controller-metrics:
+controller_metrics:
   - name: loss
     class: Loss
 controllers:
-  - name: loss-controller-wrong-os-rule
+  - name: loss_controller_wrong_os_rule
     triggers:
       - on_log
     rule: __import__('os').system('clear')