fix: correct negative sigma preds in LocallyAdaptiveCP (#53)

* LocallyAdaptiveCP: authorise negative MAD values * ops: updated setup and doc config files * doc: added experimental pytorch wrapper --------- Co-authored-by: M-Mouhcine <[email protected]>
deel-ai · Mar 20, 2024 · 2c65579 · 2c65579
1 parent 3b9e4e5
commit 2c65579
Show file tree

Hide file tree

Showing 7 changed files with 34 additions and 12 deletions.
diff --git a/deel/puncc/api/experimental.py b/deel/puncc/api/experimental.py
@@ -23,9 +23,9 @@
 """
 This module provides experimental features. Use cautiously.
 """
-import pkgutil
+import importlib.util
 
-if pkgutil.find_loader("torch") is not None:
+if importlib.util.find_spec("torch") is not None:
     import torch
 
     class TorchPredictor:

diff --git a/deel/puncc/api/nonconformity_scores.py b/deel/puncc/api/nonconformity_scores.py
@@ -30,7 +30,6 @@
 
 import numpy as np
 
-from deel.puncc.api.utils import EPSILON
 from deel.puncc.api.utils import logit_normalization_check
 from deel.puncc.api.utils import supported_types_check
 
@@ -206,7 +205,7 @@ def absolute_difference(y_pred: Iterable, y_true: Iterable) -> Iterable:
     return abs(difference(y_pred, y_true))
 
 
-def scaled_ad(Y_pred: Iterable, y_true: Iterable) -> Iterable:
+def scaled_ad(Y_pred: Iterable, y_true: Iterable, eps: float = 1e-12) -> Iterable:
     """Scaled Absolute Deviation, normalized by an estimation of the conditional
     mean absolute deviation (conditional MAD). Considering
     :math:`Y_{\\text{pred}} = (\mu_{\\text{pred}}, \sigma_{\\text{pred}})`:
@@ -218,6 +217,7 @@ def scaled_ad(Y_pred: Iterable, y_true: Iterable) -> Iterable:
     :param Iterable Y_pred: point and conditional MAD predictions.
         :math:`Y_{\\text{pred}}=(y_{\\text{pred}}, \sigma_{\\text{pred}})`
     :param Iterable y_true: true labels.
+    :param float eps: small positive value to avoid division by negative or zero.
 
     :returns: scaled absolute deviation.
     :rtype: Iterable
@@ -244,9 +244,12 @@ def scaled_ad(Y_pred: Iterable, y_true: Iterable) -> Iterable:
 
     # MAD then Scaled MAD and computed
     mean_absolute_deviation = absolute_difference(y_pred, y_true)
-    if np.any(sigma_pred < 0):
-        raise RuntimeError("All MAD predictions should be positive.")
-    return mean_absolute_deviation / (sigma_pred + EPSILON)
+    if np.any(sigma_pred + eps <= 0):
+        print("Warning: calibration points with MAD predictions"
+              " below -eps won't be used for calibration.")
+
+    nonneg = sigma_pred + eps > 0
+    return mean_absolute_deviation[nonneg] / (sigma_pred[nonneg] + eps)
 
 
 def cqr_score(Y_pred: Iterable, y_true: Iterable) -> Iterable:

diff --git a/deel/puncc/api/prediction_sets.py b/deel/puncc/api/prediction_sets.py
@@ -214,7 +214,7 @@ def constant_interval(
 
 
 def scaled_interval(
-    Y_pred: Iterable, scores_quantile: np.ndarray
+    Y_pred: Iterable, scores_quantile: np.ndarray, eps: float = 1e-12
 ) -> Tuple[np.ndarray]:
     """Scaled prediction interval centered on `y_pred`. Considering
     :math:`Y_{\\text{pred}} = (\mu_{\\text{pred}}, \sigma_{\\text{pred}})`,
@@ -229,6 +229,7 @@ def scaled_interval(
     :param Iterable y_pred: predictions.
     :param ndarray scores_quantile: quantile of nonconformity scores computed
         on a calibration set for a given :math:`\\alpha`.
+    :param float eps: small positive value to avoid singleton sets.
 
     :returns: scaled prediction intervals :math:`I`.
     :rtype: Tuple[ndarray]
@@ -249,8 +250,16 @@ def scaled_interval(
     else:
         y_pred, sigma_pred = Y_pred[:, 0], Y_pred[:, 1]
 
-    y_lo = y_pred - scores_quantile * sigma_pred
-    y_hi = y_pred + scores_quantile * sigma_pred
+    if np.any(sigma_pred + eps <= 0):
+        print("Warning: test points with MAD predictions below -eps"
+              " will have infinite sized prediction intervals.")
+
+    fints = sigma_pred + eps > 0
+    y_lo, y_hi = np.zeros_like(y_pred), np.zeros_like(y_pred)
+    y_lo[fints] = y_pred[fints] - scores_quantile * (sigma_pred[fints] + eps)
+    y_hi[fints] = y_pred[fints] + scores_quantile * (sigma_pred[fints] + eps)
+    y_lo[~fints] = -np.inf
+    y_hi[~fints] = np.inf
     return y_lo, y_hi
 
 

diff --git a/docs/source/api.rst b/docs/source/api.rst
@@ -25,6 +25,7 @@ API's Modules
 
     conformalization
     prediction
+    experimental
     calibration
     splitting
     utils

diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -26,7 +26,7 @@
 author = "Mouhcine Mendil, Luca Mossina and Joseba Dalmau"
 
 # The full version, including alpha/beta/rc tags
-release = "0.7.6"
+release = "0.7.7"
 
 
 # -- General configuration ---------------------------------------------------

diff --git a/docs/source/experimental.rst b/docs/source/experimental.rst
@@ -0,0 +1,9 @@
+.. _experimental:
+
+Experimental
+============
+
+.. automodule:: experimental
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/setup.py b/setup.py
@@ -61,7 +61,7 @@
 
 setuptools.setup(
     name="puncc",
-    version="0.7.6",
+    version="0.7.7",
     author=", ".join(["Mouhcine Mendil", "Luca Mossina", "Joseba Dalmau"]),
     author_email=", ".join(
         [