trying with multi logic

nomadic-ml · Sep 11, 2024 · 90b3c61 · 90b3c61
1 parent 86a3ab0
commit 90b3c61
Showing 1 changed file with 87 additions and 39 deletions.
diff --git a/nomadic/experiment/base.py b/nomadic/experiment/base.py
@@ -496,58 +496,106 @@ def _calculate_mean_score(
         if not eval_results:
             print("Warning: No evaluation results provided.")
             return {}
-    
+
         scores_by_params = {}
-
+
+        # Determine whether to use new logic based on evaluator method
+        use_new_logic = (
+            isinstance(self.evaluator, dict)
+            and self.evaluator.get("method") == "custom_evaluate"
+        )
+
         for result in eval_results:
-            # Check for the standard format with a single "score"
-            if isinstance(result, dict) and "score" in result:
-                score = result["score"]
-                if not isinstance(score, (int, float)):
-                    print(f"Warning: Invalid score type: {type(score)}. Expected int or float.")
-                    continue
-
-                params = result.get("params", {})
-                param_key = tuple(
-                    sorted((k, v) for k, v in params.items() if k in self.params)
-                )
-
-                if param_key not in scores_by_params:
-                    scores_by_params[param_key] = []
-                scores_by_params[param_key].append(score)
-
-            # Check for the new format with "overall_score"
-            elif isinstance(result, dict) and "overall_score" in result:
-                overall_score = result["overall_score"]
-                if not isinstance(overall_score, (int, float)):
-                    print(f"Warning: Invalid overall_score type: {type(overall_score)}. Expected int or float.")
+            if use_new_logic:
+                # New logic: Handle various score formats
+                if "score" in result and isinstance(result["score"], (int, float)):
+                    score = result["score"]
+                    params = result.get("params", {})
+                    param_key = tuple(
+                        sorted((k, v) for k, v in params.items() if k in self.params)
+                    )
+
+                    if param_key not in scores_by_params:
+                        scores_by_params[param_key] = []
+                    scores_by_params[param_key].append(score)
+
+                elif "overall_score" in result and isinstance(result["overall_score"], (int, float)):
+                    overall_score = result["overall_score"]
+                    params = result.get("params", {})
+                    param_key = tuple(
+                        sorted((k, v) for k, v in params.items() if k in self.params)
+                    )
+
+                    if param_key not in scores_by_params:
+                        scores_by_params[param_key] = []
+                    scores_by_params[param_key].append(overall_score)
+
+                elif "scores" in result and isinstance(result["scores"], dict):
+                    overall_score = result.get("overall_score")
+                    if overall_score is not None and isinstance(overall_score, (int, float)):
+                        params = result.get("params", {})
+                        param_key = tuple(
+                            sorted((k, v) for k, v in params.items() if k in self.params)
+                        )
+
+                        if param_key not in scores_by_params:
+                            scores_by_params[param_key] = []
+                        scores_by_params[param_key].append(overall_score)
+                    else:
+                        print(f"Warning: 'scores' present but 'overall_score' is missing or invalid: {result}")
+
+                else:
+                    print(f"Warning: Unexpected result format: {result}")
                     continue
-
-                params = result.get("params", {})
-                param_key = tuple(
-                    sorted((k, v) for k, v in params.items() if k in self.params)
-                )
-
-                if param_key not in scores_by_params:
-                    scores_by_params[param_key] = []
-                scores_by_params[param_key].append(overall_score)
-
-            # Handle unexpected formats
+
             else:
-                print(f"Warning: Unexpected result format: {result}")
-                continue
-
+                # Old logic: Check for the standard format with a single "score"
+                if isinstance(result, dict) and "score" in result:
+                    score = result["score"]
+                    if not isinstance(score, (int, float)):
+                        print(f"Warning: Invalid score type: {type(score)}. Expected int or float.")
+                        continue
+
+                    params = result.get("params", {})
+                    param_key = tuple(
+                        sorted((k, v) for k, v in params.items() if k in self.params)
+                    )
+
+                    if param_key not in scores_by_params:
+                        scores_by_params[param_key] = []
+                    scores_by_params[param_key].append(score)
+
+                elif isinstance(result, dict) and "overall_score" in result:
+                    overall_score = result["overall_score"]
+                    if not isinstance(overall_score, (int, float)):
+                        print(f"Warning: Invalid overall_score type: {type(overall_score)}. Expected int or float.")
+                        continue
+
+                    params = result.get("params", {})
+                    param_key = tuple(
+                        sorted((k, v) for k, v in params.items() if k in self.params)
+                    )
+
+                    if param_key not in scores_by_params:
+                        scores_by_params[param_key] = []
+                    scores_by_params[param_key].append(overall_score)
+
+                else:
+                    print(f"Warning: Unexpected result format: {result}")
+                    continue
+
         # Calculate mean scores for all parameter combinations
         mean_scores = {
             str(param_key): sum(scores) / len(scores)
             for param_key, scores in scores_by_params.items()
         }
-    
+
         print("Debug: Calculated mean_scores =", mean_scores)
-    
+
         return mean_scores
 
 
+
     def _setup_tuner(self, param_dict: Dict[str, Any], param_function: Callable):
         if not self.tuner:
             if self.enable_logging: