Merge pull request #9 from RoySadaka/threshold_checker_improvement

Threshold checker improvement

Author: NaorHaba

lpd/callbacks/callback_monitor.py

@@ -32,8 +32,7 @@ def __init__(self, monitor_type: MonitorType,
         self.monitor_mode = monitor_mode
         self.threshold_checker = AbsoluteThresholdChecker(monitor_mode) if threshold_checker is None else threshold_checker
         self.metric_name = metric_name
-        self.minimum = None
-        self.maximum = None
+        self.best = None
         self.previous = None
         self.description = self._get_description()
         self._track_invoked = False
@@ -45,7 +44,7 @@ def _get_description(self):
         return desc
 
     def _get_best(self):
-        return self.minimum if self.monitor_mode == MonitorMode.MIN else self.maximum
+        return self.best
 
     def track(self, callback_context: CallbackContext) -> CallbackMonitorResult:
         c = callback_context #READABILITY DOWN THE ROAD
@@ -70,8 +69,10 @@ def track(self, callback_context: CallbackContext) -> CallbackMonitorResult:
             value_to_consider = metrics_to_consider[self.metric_name]
 
         if not self._track_invoked:
-            self.minimum = -torch.log(torch.zeros_like(value_to_consider))  # [[inf,...,inf]]
-            self.maximum = torch.log(torch.zeros_like(value_to_consider)) # [[-inf,...,-inf]]
+            if self.monitor_mode == MonitorMode.MIN:
+                self.best = -torch.log(torch.zeros_like(value_to_consider))  # [[inf,...,inf]]
+            elif self.monitor_mode == MonitorMode.MAX:
+                self.best = torch.log(torch.zeros_like(value_to_consider)) # [[-inf,...,-inf]]
             self.previous = self._get_best()
             self._track_invoked = True
 
@@ -81,19 +82,18 @@ def track(self, callback_context: CallbackContext) -> CallbackMonitorResult:
         change_from_previous = value_to_consider - self.previous
         curr_best = self._get_best()
         change_from_best = value_to_consider - curr_best
-        self.minimum = torch.min(self.minimum, value_to_consider)
-        self.maximum = torch.max(self.maximum, value_to_consider)
         curr_previous = self.previous
         self.previous = value_to_consider
         did_improve = False # UNLESS SAID OTHERWISE
-        new_best = self._get_best()
+        new_best = curr_best # UNLESS SAID OTHERWISE
         name = self.metric_name if self.metric_name else 'loss'
 
         if len(value_to_consider.shape) == 0 or  \
            (len(value_to_consider.shape) == 1 and value_to_consider.shape[0] == 1):
             if self.threshold_checker(new_value=value_to_consider, old_value=curr_best):
                 did_improve = True
                 self.patience_countdown = self.patience
+                self.best = new_best = value_to_consider
         else:
             if self.patience != inf:
                 raise ValueError("[CallbackMonitor] - can't monitor patience for metric that has multiple values")

lpd/metrics/mock_metric.py

@@ -0,0 +1,16 @@
+import torch
+
+from lpd.enums import MetricMethod
+from lpd.metrics import MetricBase
+
+
+class MockMetric(MetricBase):
+    def __init__(self, mock_value, name: str):
+        super(MockMetric, self).__init__(name=name, metric_method=MetricMethod.LAST)
+        self.mock_value = mock_value
+
+    def __call__(self, y_pred: torch.Tensor, y_true: torch.Tensor):
+        return torch.FloatTensor([self.mock_value])
+
+    def set_mock_value(self, mock_value):
+        self.mock_value = mock_value

lpd/trainer_stats.py

@@ -28,15 +28,11 @@ def add_value(self, value, count):
         elif self.metric_method == MetricMethod.SUM:
             self.sum += value
             self.count += count
-        
-        elif self.metric_method == MetricMethod.LAST:
-            self.sum = value
-            self.count = 1
 
         self.last = value
 
     def get_value(self):
-        if self.count == 0:
+        if self.last is None:
             return torch.tensor(0.0)
 
         if self.metric_method == MetricMethod.MEAN:
@@ -46,7 +42,7 @@ def get_value(self):
             return self.sum
 
         elif self.metric_method == MetricMethod.LAST:
-            return self.sum
+            return self.last
 
 class StatsResult():
     def __init__(self, trainer_name, stats):

tests/test_callbacks.py

@@ -3,14 +3,18 @@
 
 import torch.optim as optim
 import torch.nn as nn
+
+from lpd.metrics.mock_metric import MockMetric
 from lpd.trainer import Trainer
-from lpd.callbacks import StatsPrint, SchedulerStep, LossOptimizerHandler, ModelCheckPoint, CallbackMonitor
+from lpd.callbacks import StatsPrint, SchedulerStep, LossOptimizerHandler, ModelCheckPoint, CallbackMonitor, \
+    CallbackContext
 from lpd.extensions.custom_schedulers import KerasDecay
 from lpd.enums import Phase, State, MonitorType, StatsType, MonitorMode
 from lpd.metrics import BinaryAccuracyWithLogits, CategoricalAccuracyWithLogits
 import lpd.utils.torch_utils as tu
 import lpd.utils.general_utils as gu
 import examples.utils as eu
+from lpd.utils.threshold_checker import AbsoluteThresholdChecker
 
 
 class TestCallbacks(unittest.TestCase):
@@ -24,4 +28,63 @@ def test_stats_print_validations(self):
         StatsPrint(train_metrics_monitors=CallbackMonitor(monitor_type=MonitorType.METRIC,
                                                           stats_type=StatsType.TRAIN,
                                                           monitor_mode=MonitorMode.MAX,
-                                                          metric_name='Accuracy'))
+                                                          metric_name='Accuracy'))
+
+    def test_did_improve_gradually(self):
+        gu.seed_all(42)
+
+        device = tu.get_gpu_device_if_available()
+
+        model = eu.get_basic_model(10, 10, 10).to(device)
+
+        loss_func = nn.CrossEntropyLoss().to(device)
+
+        optimizer = optim.Adam(model.parameters(), lr=1e-4)
+
+        scheduler = KerasDecay(optimizer, 0.0001, last_step=-1)
+
+        metrics = MockMetric(0.0, 'mock_metric')
+
+        callbacks = [
+            LossOptimizerHandler()
+        ]
+
+        data_loader = eu.examples_data_generator(10, 10, 10, category_out=True)
+        data_loader_steps = 1
+
+        trainer = Trainer(model=model,
+                          device=device,
+                          loss_func=loss_func,
+                          optimizer=optimizer,
+                          scheduler=scheduler,
+                          metrics=metrics,
+                          train_data_loader=data_loader,
+                          val_data_loader=data_loader,
+                          train_steps=data_loader_steps,
+                          val_steps=data_loader_steps,
+                          callbacks=callbacks,
+                          name='Trainer-Test')
+
+        mock_values = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
+        threshold = 0.99
+        sp = StatsPrint(train_metrics_monitors=CallbackMonitor(monitor_type=MonitorType.METRIC,
+                                                               stats_type=StatsType.TRAIN,
+                                                               monitor_mode=MonitorMode.MAX,
+                                                               threshold_checker=AbsoluteThresholdChecker(MonitorMode.MAX,
+                                                                                                          threshold),
+                                                               metric_name='mock_metric'))
+
+        trainer.train(1)  # IMPROVE inf TO 0.0
+        res = sp.train_metrics_monitors[0].track(CallbackContext(trainer))
+        assert res.did_improve
+
+        for mock_value in mock_values:
+            metrics.set_mock_value(mock_value)
+            trainer.train(1)
+            res = sp.train_metrics_monitors[0].track(CallbackContext(trainer))
+            assert not res.did_improve
+
+        metrics.set_mock_value(1.0)  # IMPROVE 0.0 TO 1.0 (> 0.99)
+        trainer.train(1)
+        res = sp.train_metrics_monitors[0].track(CallbackContext(trainer))
+        assert res.did_improve