update_static_tracking_metrics

Nidelva3D/src/Agents/Agent_adaptive.py

@@ -18,7 +18,7 @@
 from AUVSimulator.AUVSimulator import AUVSimulator
 from Visualiser.Visualiser_myopic import Visualiser
 from usr_func.checkfolder import checkfolder
-from sklearn.metrics import mean_squared_error
+from sklearn.metrics import mean_squared_error, roc_auc_score
 from scipy.stats import norm
 import numpy as np
 import time
@@ -59,9 +59,13 @@ def __init__(self, kernel: str = "GMRF", num_steps: int = 5,
         self.__threshold = self.myopic.kernel.get_threshold()
         self.__ibv = np.zeros([self.__num_steps, 1])
         self.__vr = np.zeros([self.__num_steps, 1])
-        self.__rmse = np.zeros([self.__num_steps, 1])
-        self.__ce = np.zeros([self.__num_steps, 1])
+        self.__rmse_temporal = np.zeros([self.__num_steps, 1])
+        self.__rmse_static = np.zeros([self.__num_steps, 1])
+        self.__ce_temporal = np.zeros([self.__num_steps, 1])
+        self.__ce_static = np.zeros([self.__num_steps, 1])
         self.__corr_coef = np.zeros([self.__num_steps, 1])
+        self.__auc_temporal = np.zeros([self.__num_steps, 1])
+        self.__auc_static = np.zeros([self.__num_steps, 1])
         self.update_metrics()
 
         self.debug = debug
@@ -184,15 +188,24 @@ def run(self):
     def update_metrics(self) -> None:
         mu = self.myopic.kernel.get_mu()
         sigma_diag = self.myopic.kernel.get_mvar()
-        self.mu_truth = self.auv.ctd.get_salinity_at_dt_loc(dt=0, loc=self.grid)
         self.__ibv[self.__counter] = self.__get_ibv(self.__threshold, mu, sigma_diag)
         self.__vr[self.__counter] = np.sum(sigma_diag)
 
         mu[mu < 0] = 0
-        self.__rmse[self.__counter] = mean_squared_error(self.mu_truth.flatten(), mu.flatten(), squared=False)
+
+        # s0, update static metrics
+        self.__rmse_static[self.__counter] = mean_squared_error(self.mu_truth.flatten(), mu.flatten(), squared=False)
+        self.__ce_static[self.__counter] = self.__cal_classification_error(self.mu_truth.flatten(),
+                                                                             mu.flatten(), self.__threshold)
+        self.__auc_static[self.__counter] = self.__cal_auc_roc(self.__threshold, mu, self.mu_truth, sigma_diag)
+
+        # s1, update temporal metrics
+        self.mu_truth = self.auv.ctd.get_salinity_at_dt_loc(dt=0, loc=self.grid)
+        self.__rmse_temporal[self.__counter] = mean_squared_error(self.mu_truth.flatten(), mu.flatten(), squared=False)
         self.__corr_coef[self.__counter] = self.__cal_corr_coef(self.mu_truth.flatten(), mu.flatten())
-        self.__ce[self.__counter] = self.__cal_classification_error(self.mu_truth.flatten(),
-                                                                    mu.flatten(), self.__threshold)
+        self.__ce_temporal[self.__counter] = self.__cal_classification_error(self.mu_truth.flatten(),
+                                                                             mu.flatten(), self.__threshold)
+        self.__auc_temporal[self.__counter] = self.__cal_auc_roc(self.__threshold, mu, self.mu_truth, sigma_diag)
 
     @staticmethod
     def __cal_corr_coef(x, y) -> float:
@@ -220,6 +233,16 @@ def __cal_classification_error(x, y, threshold) -> float:
         CE = np.sum(np.abs(X - Y)) / len(X)
         return CE
 
+    @staticmethod
+    def __cal_auc_roc(threshold: float, mu: np.ndarray, mu_truth: np.ndarray, sigma_diag: np.ndarray) -> float:
+        """
+        Calculate the area under the curve between two vectors.
+        """
+        p = norm.cdf(threshold, mu.squeeze(), np.sqrt(sigma_diag))
+        truth_labels = np.where(mu_truth < threshold, 1, 0)
+        auc_roc = roc_auc_score(truth_labels, p)
+        return auc_roc
+
     @staticmethod
     def __get_ibv(threshold: float, mu: np.ndarray, sigma_diag: np.ndarray) -> np.ndarray:
         """ !!! Be careful with dimensions, it can lead to serious problems.
@@ -237,7 +260,8 @@ def get_counter(self):
         return self.__counter
 
     def get_metrics(self) -> tuple:
-        return self.__ibv, self.__vr, self.__rmse, self.__corr_coef, self.__ce
+        return self.__ibv, self.__vr, self.__rmse_temporal, self.__corr_coef, self.__ce_temporal, \
+            self.__auc_temporal, self.__rmse_static, self.__ce_static, self.__auc_static
 
 
 if __name__ == "__main__":

Nidelva3D/src/GMRF/spde.py

@@ -128,6 +128,6 @@ def setThreshold(self):
     def getThreshold(self) -> np.ndarray:
         """ Get updated threshold """
         ind = np.load(FILEPATH + 'models/boundary.npy')
-        threshold = self.mu[ind].mean()
+        threshold = 27.5  # self.mu[ind].mean()
         return threshold
 

Nidelva3D/src/Simulators/Simulator.py

@@ -29,12 +29,27 @@ def __init__(self, num_steps: int = 5, random_seed: int = 0, temporal_truth: boo
     def run(self) -> 'pd.DataFrame':
         self.grf_agent.run()
         self.gmrf_agent.run()
-        self.ibv_grf, self.vr_grf, self.rmse_grf, self.corr_grf, self.ce_grf = self.grf_agent.get_metrics()
-        self.ibv_gmrf, self.vr_gmrf, self.rmse_gmrf, self.corr_gmrf, self.ce_gmrf = self.gmrf_agent.get_metrics()
-        df = np.hstack((self.ibv_grf, self.vr_grf, self.rmse_grf, self.corr_grf, self.ce_grf,
-                        self.ibv_gmrf, self.vr_gmrf, self.rmse_gmrf, self.corr_gmrf, self.ce_gmrf))
-        df = pd.DataFrame(df, columns=["ibv_grf", "vr_grf", "rmse_grf", "corr_grf", "ce_grf",
-                                       "ibv_gmrf", "vr_gmrf", "rmse_gmrf", "corr_gmrf", "ce_gmrf"])
+        (self.ibv_grf, self.vr_grf, self.rmse_grf_temporal, self.corr_grf,
+         self.ce_grf_temporal, self.auc_grf_temporal, self.rmse_grf_static,
+         self.ce_grf_static, self.auc_grf_static) = self.grf_agent.get_metrics()
+
+        (self.ibv_gmrf, self.vr_gmrf, self.rmse_gmrf_temporal, self.corr_gmrf,
+         self.ce_gmrf_temporal, self.auc_gmrf_temporal, self.rmse_gmrf_static,
+         self.ce_gmrf_static, self.auc_gmrf_static) = self.gmrf_agent.get_metrics()
+
+        df = np.hstack((self.ibv_grf, self.vr_grf, self.rmse_grf_temporal, self.corr_grf,
+                        self.ce_grf_temporal, self.auc_grf_temporal, self.rmse_grf_static,
+                        self.ce_grf_static, self.auc_grf_static,
+                        self.ibv_gmrf, self.vr_gmrf, self.rmse_gmrf_temporal, self.corr_gmrf,
+                        self.ce_gmrf_temporal, self.auc_gmrf_temporal, self.rmse_gmrf_static,
+                        self.ce_gmrf_static, self.auc_gmrf_static))
+
+        df = pd.DataFrame(df, columns=["ibv_grf", "vr_grf", "rmse_grf_temporal", "corr_grf",
+                                        "ce_grf_temporal", "auc_grf_temporal", "rmse_grf_static",
+                                        "ce_grf_static", "auc_grf_static",
+                                        "ibv_gmrf", "vr_gmrf", "rmse_gmrf_temporal", "corr_gmrf",
+                                        "ce_gmrf_temporal", "auc_gmrf_temporal", "rmse_gmrf_static",
+                                        "ce_gmrf_static", "auc_gmrf_static"])
         return df
 
 

Nidelva3D/src/tests/test_agent_adaptive.py

@@ -16,7 +16,7 @@ class TestAgent(TestCase):
 
     def setUp(self) -> None:
 
-        random_seed = 21
+        random_seed = 22
         num_steps = 30
         debug = True
 
@@ -26,7 +26,6 @@ def setUp(self) -> None:
         kernel = "GMRF"
         self.agent_gmrf = Agent(kernel=kernel, num_steps=num_steps, random_seed=random_seed, debug=debug)
 
-
     # def test_compare_gmrf_grf(self) -> None:
     #     wp_grf = self.agent_grf.myopic.waypoint_graph.get_waypoints()
     #     wp_gmrf = self.agent_gmrf.myopic.waypoint_graph.get_waypoints()

Nidelva3D/src/tests/test_simulator.py

@@ -14,8 +14,8 @@
 class TestSimulator(TestCase):
 
     def setUp(self) -> None:
-        num_steps = 20
-        random_seed = 19
+        num_steps = 30
+        random_seed = 25
         debug = True
         temporal_truth = True
         self.simulator = Simulator(num_steps=num_steps, random_seed=random_seed,
@@ -28,6 +28,7 @@ def test_run_simulator(self) -> None:
         rmse = np.stack((df["rmse_grf"], df["rmse_gmrf"]), axis=1)
         corr = np.stack((df["corr_grf"], df["corr_gmrf"]), axis=1)
         ce = np.stack((df["ce_grf"], df["ce_gmrf"]), axis=1)
+        auc = np.stack((df["auc_grf"], df["auc_gmrf"]), axis=1)
 
         plt.figure(figsize=(40, 8))
         plt.subplot(151)
@@ -52,10 +53,10 @@ def test_run_simulator(self) -> None:
         plt.legend()
         plt.title("RMSE")
         plt.subplot(154)
-        plt.plot(corr[:, 0], label="GRF")
-        plt.plot(corr[:, 1], label="GMRF")
+        plt.plot(auc[:, 0], label="GRF")
+        plt.plot(auc[:, 1], label="GMRF")
         plt.legend()
-        plt.title("Correlation")
+        plt.title("AUC")
         plt.subplot(155)
         plt.plot(ce[:, 0], label="GRF")
         plt.plot(ce[:, 1], label="GMRF")