feat: export evaluation results to csv (#14)

Also added the new path for outputs in the animation code.

src/common/data_models.py

@@ -60,6 +60,9 @@ def save_embeddings(self, embeddings: Embeddings, overwrite: bool = False) -> bo
     def build_embeddings(self) -> Embeddings:
         return Embeddings.from_embedding_definition(self.model, self.dataset)
 
+    def __str__(self):
+        return self.model + "_" + self.dataset
+
 
 if __name__ == "__main__":
     def_ = EmbeddingDefinition(

src/constants.py

@@ -6,6 +6,7 @@
 load_dotenv()
 
 _CACHE_PATH = Path(os.environ.get("CLIP_CACHE_PATH", ".cache"))
+_OUTPUT_PATH = Path(os.environ.get("OUTPUT_PATH", "output"))
 
 
 class PROJECT_PATHS:
@@ -17,3 +18,8 @@ class NPZ_KEYS:
     IMAGE_EMBEDDINGS = "image_embeddings"
     CLASS_EMBEDDINGS = "class_embeddings"
     LABELS = "labels"
+
+
+class OUTPUT_PATH:
+    ANIMATIONS = _OUTPUT_PATH / "animations"
+    EVALUATIONS = _OUTPUT_PATH / "evaluations"

src/evaluation/evaluator.py

@@ -1,6 +1,10 @@
+import csv
+from datetime import datetime
+
 import numpy as np
 
 from src.common.data_models import EmbeddingDefinition, Embeddings
+from src.constants import OUTPUT_PATH
 from src.evaluation import (
     ClassificationModel,
     LinearProbeClassifier,
@@ -11,7 +15,7 @@
 
 
 def run_evaluation(
-    classifiers: ClassificationModel,
+    classifiers: list[type[ClassificationModel]],
     embedding_definitions: list[EmbeddingDefinition],
     seed: int = 42,
     train_split: float = 0.7,  # TODO: This is very much out of the blue
@@ -55,9 +59,29 @@ def run_evaluation(
     return embeddings_performance
 
 
+def export_evaluation_to_csv(
+    embedding_definitions: list[EmbeddingDefinition],
+    embeddings_performance: list[dict[str, float]],
+) -> None:
+    ts = datetime.now()
+    results_file = OUTPUT_PATH.EVALUATIONS / f"eval_{ts.isoformat()}.csv"
+    results_file.parent.mkdir(parents=True, exist_ok=True)  # Ensure that parent folder exists
+
+    headers = ["Model", "Dataset", "Classifier", "Accuracy"]
+    with open(results_file.as_posix(), "w", newline="") as csvfile:
+        writer = csv.writer(csvfile)
+        writer.writerow(headers)
+
+        for def_, perf in zip(embedding_definitions, embeddings_performance, strict=True):
+            def_: EmbeddingDefinition
+            for classifier_title, accuracy in perf.items():
+                writer.writerow([def_.model, def_.dataset, classifier_title, accuracy])
+
+
 if __name__ == "__main__":
     models = [ZeroShotClassifier, LinearProbeClassifier, WeightedKNNClassifier]
     defs = [d for k, v in read_all_cached_embeddings().items() for d in v]
     print(defs)
     performances = run_evaluation(models, defs)
+    export_evaluation_to_csv(defs, performances)
     print(performances)

src/plotting/animation.py

@@ -1,11 +1,12 @@
-import pathlib
+from datetime import datetime
 
 import matplotlib.animation as animation
 import matplotlib.pyplot as plt
 import numpy as np
-from reduction import PCAReducer, UMAPReducer
+from reduction import UMAPReducer
 
-from src.common import EmbeddingArray, EmbeddingDefinition, Embeddings
+from src.common import EmbeddingDefinition
+from src.constants import OUTPUT_PATH
 
 
 def update_plot(t):
@@ -29,10 +30,8 @@ def update_plot(t):
     # plt.scatter(reduced_2[:, 0], reduced_2[:, 1], c=embd_2.labels, facecolors="none")
     plt.legend(*scat.legend_elements(), loc="upper right")
     anim = animation.FuncAnimation(fig, update_plot, frames=np.arange(0, 1, 0.05))
-    defn_str = lambda x: f"{x.model}_{x.dataset}"
-    filename = f"Transition_{defn_str(defn_1)}_{defn_str(defn_2)}.gif"
-    OUTPUT_DIR = pathlib.Path("./output/animations/")
-    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
-    OUTPUT_PATH = OUTPUT_DIR / filename
-    anim.save(OUTPUT_PATH)
+    ts = datetime.now()
+    animation_file = OUTPUT_PATH.ANIMATIONS / f"transition_{defn_1}-{defn_2}_{ts.isoformat()}.gif"
+    animation_file.parent.mkdir(parents=True, exist_ok=True)  # Ensure that parent folder exists
+    anim.save(animation_file)
     plt.show()