Merge pull request #95 from wwu-mmll/feature/score_train_v2

Feature/score train v2

Author: jernsting

photonai/base/hyperpipe.py

@@ -297,7 +297,9 @@ def __init__(self, name: Optional[str],
                  cache_folder: str = None,
                  nr_of_processes: int = 1,
                  multi_threading: bool = True,
-                 allow_multidim_targets: bool = False):
+                 allow_multidim_targets: bool = False,
+                 raise_error: bool = False,
+                 score_train: bool = True):
         """
         Initialize the object.
 
@@ -420,6 +422,12 @@ def __init__(self, name: Optional[str],
             allow_multidim_targets:
                 Allows multidimensional targets.
 
+            score_train:
+                metrics for the train-set are only calculated if score_train is true.
+
+            raise_error:
+                if true, errors in the inner fold are raised instead of suppressed as warnings.
+
         """
 
         self.name = re.sub(r'\W+', '', name)
@@ -514,6 +522,8 @@ def __init__(self, name: Optional[str],
         self.permutation_id = permutation_id
         self.allow_multidim_targets = allow_multidim_targets
         self.is_final_fit = False
+        self.score_train = score_train
+        self.raise_error = raise_error
 
         # ====================== Random Seed ===========================
         self.random_state = random_seed
@@ -941,8 +951,6 @@ def _finalize_optimization(self):
                 else:
                     self.results.best_config_feature_importances = feature_importances
 
-                    self.results.best_config_feature_importances = feature_importances
-
                     # write backmapping file only if optimum_pipes inverse_transform works completely.
                     # restriction: only a faulty inverse_transform is considered, missing ones are further ignored.
                     # with warnings.catch_warnings(record=True) as w:
@@ -1087,7 +1095,9 @@ def fit(self, data: np.ndarray, targets: np.ndarray, **kwargs):
                                                            cache_folder=self.cache_folder,
                                                            cache_updater=self.recursive_cache_folder_propagation,
                                                            dummy_estimator=dummy_estimator,
-                                                           result_obj=outer_fold)
+                                                           result_obj=outer_fold,
+                                                           score_train=self.score_train,
+                                                           raise_error=self.raise_error)
                     # 2. monitor outputs
                     self.results.outer_folds.append(outer_fold)
 

photonai/processing/inner_folds.py

@@ -66,7 +66,8 @@ def __init__(self, pipe_ctor, specific_config: dict, optimization_infos,
                  training: bool = False,
                  cache_folder=None,
                  cache_updater=None,
-                 scorer: Scorer = None):
+                 scorer: Scorer = None,
+                 score_train: bool = True):
 
         self.params = specific_config
         self.pipe = pipe_ctor
@@ -81,6 +82,7 @@ def __init__(self, pipe_ctor, specific_config: dict, optimization_infos,
 
         self.raise_error = raise_error
         self.training = training
+        self.score_train = score_train
 
     def fit(self, X, y, **kwargs):
         """Iterates over cross-validation folds and trains the pipeline,
@@ -136,7 +138,8 @@ def fit(self, X, y, **kwargs):
                                                                                            kwargs_cv_train),
                                                        test_data=InnerFoldManager.JobData(test_X, test_y, test,
                                                                                           kwargs_cv_test),
-                                                       scorer=self.scorer)
+                                                       scorer=self.scorer,
+                                                       score_train=self.score_train)
 
                 # only for unparallel processing
                 # inform children in which inner fold we are
@@ -224,7 +227,8 @@ def compute_learning_curves(self, new_pipe, train_X, train_y, train, kwargs_cv_t
                                        callbacks=self.optimization_constraints,
                                        train_data=self.JobData(train_cut_X, train_cut_y, train_cut, train_cut_kwargs),
                                        test_data=self.JobData(test_X, test_y, test, kwargs_cv_test),
-                                       scorer=self.scorer)
+                                       scorer=self.scorer,
+                                       score_train=self.score_train)
             curr_test_cut, curr_train_cut = InnerFoldManager.fit_and_score(job_data)
             learning_curves.append([self.cross_validation_infos.learning_curves_cut.values[i], curr_test_cut.metrics,
                                     curr_train_cut.metrics])
@@ -239,14 +243,15 @@ def __init__(self, X, y, indices, cv_kwargs):
 
     class InnerCVJob:
 
-        def __init__(self, pipe, config, metrics, callbacks, train_data, test_data, scorer):
+        def __init__(self, pipe, config, metrics, callbacks, train_data, test_data, scorer, score_train):
             self.pipe = pipe
             self.config = config
             self.metrics = metrics
             self.callbacks = callbacks
             self.train_data = train_data
             self.test_data = test_data
             self.scorer = scorer
+            self.score_train = score_train
 
     @staticmethod
     def update_config_item_with_inner_fold(config_item, fold_cnt, curr_train_fold, curr_test_fold, time_monitor,
@@ -344,27 +349,28 @@ def fit_and_score(job: InnerCVJob):
         # start fitting
         pipe.fit(job.train_data.X, job.train_data.y, **job.train_data.cv_kwargs)
 
-        logger.debug('Scoring Training Data')
+        logger.debug('Scoring Test Data')
 
         # score test data
         curr_test_fold = InnerFoldManager.score(pipe, job.test_data.X, job.test_data.y, job.metrics,
                                                 indices=job.test_data.indices,
                                                 scorer=job.scorer,
                                                 **job.test_data.cv_kwargs)
 
-        logger.debug('Scoring Test Data')
+        logger.debug('Scoring Training Data')
         # score train data
         curr_train_fold = InnerFoldManager.score(pipe, job.train_data.X, job.train_data.y, job.metrics,
-                                                 indices=job.train_data.indices,
-                                                 training=True,
-                                                 scorer=job.scorer, **job.train_data.cv_kwargs)
+                                                indices=job.train_data.indices,
+                                                training=True,
+                                                score_train=job.score_train,
+                                                scorer=job.scorer, **job.train_data.cv_kwargs)
 
         return curr_test_fold, curr_train_fold
 
     @staticmethod
     def score(estimator, X, y_true, metrics, indices=[],
               calculate_metrics: bool = True, training: bool = False,
-              scorer: Scorer = None, **kwargs):
+              dummy: bool = False, scorer: Scorer = None, score_train=True, **kwargs):
         """Uses the pipeline to predict the given data,
         compare it to the truth values and calculate metrics
 
@@ -410,7 +416,18 @@ def score(estimator, X, y_true, metrics, indices=[],
 
         output_metrics = {}
 
-        if not training:
+        if training and not score_train:
+            scores = {}
+            for metric in list(metrics.keys()):
+                scores[metric] = 0
+            return MDBScoreInformation(metrics=scores,
+                                        score_duration=0,
+                                        y_pred=list(np.zeros_like(y_true)),
+                                        y_true=list(y_true),
+                                        indices=np.asarray(indices).tolist(),
+                                        probabilities=[])
+
+        if not training or (training and dummy):
             y_pred = estimator.predict(X, **kwargs)
         else:
             X, y_true_new, kwargs_new = estimator.transform(X, y_true, **kwargs)

photonai/processing/metrics.py

@@ -29,9 +29,9 @@ class Scorer:
         'precision': ('sklearn.metrics', 'precision_score', 'score'),
         'recall': ('sklearn.metrics', 'recall_score', 'score'),
         'auc': ('sklearn.metrics', 'roc_auc_score', 'score'),
-        'sensitivity': ('photonai.processing.metrics', 'sensitivity', 'score'),
+        'sensitivity': ('sklearn.metrics', 'recall_score', 'score'),
         'specificity': ('photonai.processing.metrics', 'specificity', 'score'),
-        'balanced_accuracy': ('photonai.processing.metrics', 'balanced_accuracy', 'score'),
+        'balanced_accuracy': ('sklearn.metrics', 'balanced_accuracy_score', 'score'),
         'categorical_accuracy': ('photonai.processing.metrics', 'categorical_accuracy_score', 'score'),
 
         # Regression

photonai/processing/outer_folds.py

@@ -63,14 +63,18 @@ def __init__(self, pipe,
                  cache_folder=None,
                  cache_updater=None,
                  dummy_estimator=None,
-                 result_obj=None):
+                 result_obj=None,
+                 raise_error=False,
+                 score_train: bool = True):
         self.outer_fold_id = outer_fold_id
         self.cross_validation_info = cross_validation_info
         self.scorer = Scorer(optimization_info.metrics)
         self.optimization_info = optimization_info
         self._pipe = pipe
         self.copy_pipe_fnc = self._pipe.copy_me
         self.dummy_estimator = dummy_estimator
+        self.score_train = score_train
+        self.raise_error = raise_error
 
         self.cache_folder = cache_folder
         self.cache_updater = cache_updater
@@ -255,6 +259,7 @@ def fit(self, X, y=None, **kwargs):
                                                          metrics=self.optimization_info.metrics,
                                                          training=True,
                                                          scorer=self.scorer,
+                                                         score_train=self.score_train,
                                                          **self._validation_kwargs)
 
                 best_config_performance_mdb.training = train_score_mdb
@@ -308,7 +313,8 @@ def objective_function(self, current_config):
                               self.cross_validation_info, self.outer_fold_id, self.constraint_objects,
                               cache_folder=self.cache_folder,
                               cache_updater=self.cache_updater,
-                              scorer=self.scorer)
+                              scorer=self.scorer,
+                              raise_error=self.raise_error)
 
         # Test the configuration cross validated by inner_cv object
         current_config_mdb = hp.fit(self._validation_X, self._validation_y, **self._validation_kwargs)
@@ -385,7 +391,10 @@ def _fit_dummy(self):
                 dummy_y = np.reshape(self._validation_y, (-1, 1))
                 self.dummy_estimator.fit(dummy_y, self._validation_y)
                 train_scores = InnerFoldManager.score(self.dummy_estimator, self._validation_X, self._validation_y,
+                                                      training=True,
+                                                      dummy=True,
                                                       metrics=self.optimization_info.metrics,
+                                                      score_train=self.score_train,
                                                       scorer=self.scorer)
 
                 # fill result tree with fold information
@@ -396,6 +405,7 @@ def _fit_dummy(self):
                     test_scores = InnerFoldManager.score(self.dummy_estimator,
                                                          self._test_X, self._test_y,
                                                          metrics=self.optimization_info.metrics,
+                                                         score_train=self.score_train,
                                                          scorer=self.scorer)
                     print_metrics("DUMMY", test_scores.metrics)
                     inner_fold.validation = test_scores

test/integration_tests/test_architecture.py

@@ -67,6 +67,7 @@ def create_hyperpipes(metrics: list = None, inner_cv=KFold(n_splits=3, shuffle=T
                          use_test_set=eval_final_performance,
                          performance_constraints=performance_constraints,
                          cache_folder=cache_folder,
+                         raise_error=True,
                          verbosity=0)
         return pipe