v0.4.11 (#7)

* v0.4.11
---------
Co-authored-by: Roy Sadaka <[email protected]>

Author: RoySadaka

CHANGELOG.txt

@@ -1,6 +1,12 @@
 Change Log
 ==========
 
+0.4.11 (08/02/2023)
+-----------------
+* Dense custom layer to support apply norm (configurable to before or after activation) 
+* StatsPrint callback to support printing best confusion matrix when at least one of the metrics is of type ``MetricConfusionMatrixBase``
+* Some minor cosmetic changes
+
 
 0.4.10 (04/02/2023)
 -----------------

README.md

@@ -25,24 +25,23 @@ There are 2 types of ``lpd`` packages available
     pip install lpd-nodeps
 ```
 
-<b>[v0.4.10-beta](https://github.com/RoySadaka/lpd/releases) Release - contains the following:</b> 
+<b>[v0.4.11-beta](https://github.com/RoySadaka/lpd/releases) Release - contains the following:</b> 
+
+* ``Dense`` custom layer to support apply norm (configurable to before or after activation) 
+* ``StatsPrint`` callback to support printing best confusion matrix when at least one of the metrics is of type ``MetricConfusionMatrixBase``
+* Some minor cosmetic changes
 
-* ``TransformerEncoderStack`` to support activation as input
-* ``PositionalEncoding`` to support more than 3 dimensions input
 
 
 Previously on lpd:
+* ``TransformerEncoderStack`` to support activation as input
+* ``PositionalEncoding`` to support more than 3 dimensions input
 * Updated Pipfile
 * Fixed confusion matrix cpu/gpu device error
 * Better handling on callbacks where apply_on_states=None (apply on all states)
 * Bug fix in case validation samples are empty
 * Bug fix in verbosity level 2 in train
 * Verbosity change in torch_utils
-* Fix to PositionalEncoding to be batch first
-* Minor change to MatMul2D, use torch.matmul instead of torch.bmm
-* Bug fix when saving full trainer that has tensorboard callback
-* Added LossOptimizerHandlerAccumulateSamples 
-* Added LossOptimizerHandlerAccumulateBatches
 
 
 ## Usage
@@ -271,7 +270,7 @@ only after the defined num of batches (or samples) were accumulated
 ```
 Output example: 
 
-![EpochSummary](https://raw.githubusercontent.com/RoySadaka/ReposMedia/main/lpd/images/epoch_summary.png)
+![EpochSummary](https://raw.githubusercontent.com/RoySadaka/ReposMedia/main/lpd/images/epoch_summary_0_4_11.png)
 
 
 

examples/confusion_matrix/train.py

@@ -50,7 +50,7 @@ def get_trainer(N, D_in, H, D_out, num_epochs, num_classes, data_loader, data_lo
 
     callbacks = [   
                     LossOptimizerHandler(),
-                    StatsPrint(print_confusion_matrix=True)
+                    StatsPrint(print_confusion_matrix_normalized=True)
                 ]
 
     trainer = Trainer(model=model, 

examples/multiple_inputs/model.py

@@ -6,7 +6,7 @@
 from lpd.extensions.custom_layers import TransformerEncoderStack, Attention, MatMul2D
 from lpd.enums import Phase, State, MonitorType, MonitorMode, StatsType
 from lpd.callbacks import StatsPrint, ModelCheckPoint, Tensorboard, EarlyStopping, SchedulerStep, LossOptimizerHandler, CallbackMonitor
-from lpd.metrics import BinaryAccuracyWithLogits, TruePositives
+from lpd.metrics import BinaryAccuracyWithLogits, TruePositives, FalsePositives
 from lpd.extensions.custom_schedulers import DoNothingToLR
 import lpd.utils.torch_utils as tu
 
@@ -82,7 +82,8 @@ def get_trainer(config,
 
     metrics = [
                            BinaryAccuracyWithLogits(name='Accuracy'),
-                           TruePositives(num_classes=2, threshold=0, name='TP')
+                           TruePositives(num_classes=2, threshold=0, name='TP'),
+                           FalsePositives(num_classes=2, threshold=0, name='FP')
                         ]
 
     callbacks = [   
@@ -94,10 +95,15 @@ def get_trainer(config,
                     EarlyStopping(apply_on_phase=Phase.EPOCH_END, 
                                   apply_on_states=State.EXTERNAL,
                                   callback_monitor=CallbackMonitor(monitor_type=MonitorType.LOSS, 
-                                                                    stats_type=StatsType.VAL, 
-                                                                    monitor_mode=MonitorMode.MIN,
-                                                                    patience=config.EARLY_STOPPING_PATIENCE)),
-                    StatsPrint(apply_on_phase=Phase.EPOCH_END, round_values_on_print_to=7, print_confusion_matrix_normalized=True),
+                                                                   stats_type=StatsType.VAL, 
+                                                                   monitor_mode=MonitorMode.MIN,
+                                                                   patience=config.EARLY_STOPPING_PATIENCE)),
+                    StatsPrint(apply_on_phase=Phase.EPOCH_END, 
+                               round_values_on_print_to=7, 
+                               print_confusion_matrix_normalized=True, 
+                               train_best_confusion_matrix_monitor=CallbackMonitor(monitor_type=MonitorType.LOSS, 
+                                                                                   stats_type=StatsType.TRAIN, 
+                                                                                   monitor_mode=MonitorMode.MIN)),
                     ModelCheckPoint(checkpoint_dir=checkpoint_dir, 
                                     checkpoint_file_name=checkpoint_file_name, 
                                     callback_monitor=CallbackMonitor(monitor_type=MonitorType.LOSS, 

lpd/callbacks/__init__.py

@@ -1,5 +1,6 @@
 from lpd.callbacks.callback_base import CallbackBase
-from lpd.callbacks.callback_monitor import CallbackMonitor, CallbackMonitorResult
+from lpd.callbacks.callback_monitor import CallbackMonitor
+from lpd.callbacks.callback_monitor_result import CallbackMonitorResult
 from lpd.callbacks.stats_print import StatsPrint
 from lpd.callbacks.model_checkpoint import ModelCheckPoint
 from lpd.callbacks.tensorboard import Tensorboard
@@ -11,6 +12,4 @@
 from lpd.callbacks.loss_optimizer_handler import LossOptimizerHandler
 from lpd.callbacks.loss_optimizer_handler_base import LossOptimizerHandlerBase
 from lpd.callbacks.loss_optimizer_handler_accumulate_batchs import LossOptimizerHandlerAccumulateBatches
-from lpd.callbacks.loss_optimizer_handler_accumulate_samples import LossOptimizerHandlerAccumulateSamples
-
-
+from lpd.callbacks.loss_optimizer_handler_accumulate_samples import LossOptimizerHandlerAccumulateSamples

lpd/callbacks/callback_monitor.py

@@ -1,3 +1,4 @@
+from lpd.callbacks.callback_monitor_result import CallbackMonitorResult
 from lpd.enums import Phase, State, MonitorType, MonitorMode, StatsType
 from lpd.callbacks.callback_context import CallbackContext
 from typing import Union, List, Optional, Dict
@@ -6,7 +7,6 @@
 
 from lpd.utils.threshold_checker import ThresholdChecker, AbsoluteThresholdChecker
 
-
 class CallbackMonitor:
     """
     Will check if the desired metric improved with support for patience
@@ -15,14 +15,14 @@ class CallbackMonitor:
                     (negative number will set to inf)
         monitor_type - e.g lpd.enums.MonitorType.LOSS
         stats_type - e.g lpd.enums.StatsType.VAL
-        monitor_mode - e.g. lpd.enums.MonitorMode.MIN, min wothh check if the metric decreased, MAX will check for increase
+        monitor_mode - e.g. lpd.enums.MonitorMode.MIN, will check if the metric decreased, MonitorMode.MAX will check for increase
         metric_name - in case of monitor_mode=lpd.enums.MonitorMode.METRIC, provide metric_name, otherwise, leave it None
         threshold_checker - to check if the criteria was met, if None, AbsoluteThresholdChecker with threshold=0.0 will be used
     """
-    def __init__(self, monitor_type: MonitorType, 
-                        stats_type: StatsType, 
+    def __init__(self, monitor_type: MonitorType,
+                        stats_type: StatsType,
                         monitor_mode: MonitorMode,
-                        patience: int=None, 
+                        patience: int=None,
                         metric_name: Optional[str]=None,
                         threshold_checker: Optional[ThresholdChecker]=None):
         self.patience = inf if patience is None or patience < 0 else patience
@@ -32,9 +32,9 @@ def __init__(self, monitor_type: MonitorType,
         self.monitor_mode = monitor_mode
         self.threshold_checker = AbsoluteThresholdChecker(monitor_mode) if threshold_checker is None else threshold_checker
         self.metric_name = metric_name
-        self.minimum = torch.tensor(inf)
-        self.maximum = torch.tensor(-inf)
-        self.previous = self._get_best()
+        self.minimum = None
+        self.maximum = None
+        self.previous = None
         self.description = self._get_description()
         self._track_invoked = False
 
@@ -47,7 +47,7 @@ def _get_description(self):
     def _get_best(self):
         return self.minimum if self.monitor_mode == MonitorMode.MIN else self.maximum
 
-    def track(self, callback_context: CallbackContext):
+    def track(self, callback_context: CallbackContext) -> CallbackMonitorResult:
         c = callback_context #READABILITY DOWN THE ROAD
 
         # EXTRACT value_to_consider
@@ -70,8 +70,9 @@ def track(self, callback_context: CallbackContext):
             value_to_consider = metrics_to_consider[self.metric_name]
 
         if not self._track_invoked:
-            self.minimum = -torch.log(torch.zeros_like(value_to_consider))  # [[inf,inf,inf,inf]]
-            self.maximum = torch.log(torch.zeros_like(value_to_consider)) # [[-inf,-inf,-inf,-inf]]
+            self.minimum = -torch.log(torch.zeros_like(value_to_consider))  # [[inf,...,inf]]
+            self.maximum = torch.log(torch.zeros_like(value_to_consider)) # [[-inf,...,-inf]]
+            self.previous = self._get_best()
             self._track_invoked = True
 
 
@@ -80,13 +81,11 @@ def track(self, callback_context: CallbackContext):
         change_from_previous = value_to_consider - self.previous
         curr_best = self._get_best()
         change_from_best = value_to_consider - curr_best
-        curr_minimum = self.minimum
-        curr_maximum = self.maximum
         self.minimum = torch.min(self.minimum, value_to_consider)
         self.maximum = torch.max(self.maximum, value_to_consider)
         curr_previous = self.previous
         self.previous = value_to_consider
-        did_improve = False
+        did_improve = False # UNLESS SAID OTHERWISE
         new_best = self._get_best()
         name = self.metric_name if self.metric_name else 'loss'
 
@@ -109,32 +108,3 @@ def track(self, callback_context: CallbackContext):
                                      patience_left=self.patience_countdown,
                                      description=self.description,
                                      name = name)
-
-
-class CallbackMonitorResult():
-    def __init__(self, did_improve: bool,
-                        new_value: float,
-                        prev_value: float,
-                        new_best: float,
-                        prev_best: float,
-                        change_from_previous: float,
-                        change_from_best: float,
-                        patience_left: int,
-                        description: str,
-                        name: str):
-        self.name = name
-        self.did_improve = did_improve
-        self.new_value = new_value
-        self.prev_value = prev_value
-        self.new_best = new_best
-        self.prev_best = prev_best
-        self.change_from_previous = change_from_previous
-        self.change_from_best = change_from_best
-        self.patience_left = patience_left
-        self.description = description
-
-    def has_improved(self):
-        return self.did_improve
-
-    def has_patience(self):
-        return self.patience_left > 0

lpd/callbacks/callback_monitor_result.py

@@ -0,0 +1,27 @@
+class CallbackMonitorResult():
+    def __init__(self, did_improve: bool,
+                        new_value: float,
+                        prev_value: float,
+                        new_best: float,
+                        prev_best: float,
+                        change_from_previous: float,
+                        change_from_best: float,
+                        patience_left: int,
+                        description: str,
+                        name: str):
+        self.name = name
+        self.did_improve = did_improve
+        self.new_value = new_value
+        self.prev_value = prev_value
+        self.new_best = new_best
+        self.prev_best = prev_best
+        self.change_from_previous = change_from_previous
+        self.change_from_best = change_from_best
+        self.patience_left = patience_left
+        self.description = description
+
+    def has_improved(self):
+        return self.did_improve
+
+    def has_patience(self):
+        return self.patience_left > 0

lpd/callbacks/scheduler_step.py

@@ -1,7 +1,6 @@
-from lpd.enums import Phase, State, MonitorType, MonitorMode, StatsType
+from lpd.enums import Phase, State
 from lpd.callbacks.callback_base import CallbackBase
 from lpd.callbacks.callback_context import CallbackContext
-from lpd.callbacks.callback_monitor import CallbackMonitorResult
 from lpd.utils.torch_utils import get_lrs_from_optimizer
 from typing import Union, List, Optional, Dict, Callable
 
@@ -12,7 +11,7 @@ class SchedulerStep(CallbackBase):
             apply_on_phase - see in CallbackBase
             apply_on_states - see in CallbackBase
             scheduler_parameters_func - Since some schedulers takes parameters in step(param1, param2...)
-                And other schedulers step() are parameterless, provide:
+                And other schedulers step() are parameter-less, provide:
                 a function (or lambda) that except CallbackContext and returns whatever information needed,
                 e.g. for scheduler that takes val_loss as parameter, initialize like this:
                     SchedulerStep(scheduler_parameters_func=lambda callback_context: callback_context.val_stats.get_loss())