Update caching to use last 4 caches

backend/app.py

@@ -23,7 +23,7 @@
 state = BackendState()
 
 
-@repeat_every(seconds=60 * 15, wait_first=True)
+@repeat_every(seconds=60 * 60, wait_first=True)
 async def repeatedly_retake_snapshot(state: BackendState) -> None:
     await state.take_pickle_snapshot()
 
@@ -35,15 +35,18 @@ def clean_cache(state: BackendState) -> None:
 
     pickles = glob.glob("pickles/*")
 
-    # check for pickle folders with less than 4 files (error in write)
+    # check for pickle folders with less than 8 files (error in write)
     incomplete_pickles = []
     for pickle in pickles:
-        if len(glob.glob(f"{pickle}/*")) < 4:
+        if len(glob.glob(f"{pickle}/*")) < 8:
             incomplete_pickles.append(pickle)
 
     for incomplete_pickle in incomplete_pickles:
         print(f"deleting {incomplete_pickle}")
-        shutil.rmtree(incomplete_pickle)
+        try:
+            shutil.rmtree(incomplete_pickle)
+        except Exception as e:
+            print(f"Error deleting {incomplete_pickle}: {e}")
 
     pickles = glob.glob("pickles/*")
 

backend/middleware/cache_middleware.py

@@ -3,7 +3,7 @@
 import hashlib
 import json
 import os
-from typing import Callable, Dict, Optional
+from typing import Callable, Dict, List, Optional
 
 from backend.state import BackendRequest
 from backend.state import BackendState
@@ -27,15 +27,15 @@ async def dispatch(self, request: BackendRequest, call_next: Callable):
             return await call_next(request)
 
         current_pickle = self.state.current_pickle_path
-        previous_pickle = self._get_previous_pickle()
+        previous_pickles = self._get_previous_pickles(4)  # Get last 4 pickles
 
         current_cache_key = self._generate_cache_key(request, current_pickle)
         current_cache_file = os.path.join(self.cache_dir, f"{current_cache_key}.json")
 
         if os.path.exists(current_cache_file):
             return self._serve_cached_response(current_cache_file, "Fresh")
 
-        if previous_pickle:
+        for previous_pickle in previous_pickles:
             previous_cache_key = self._generate_cache_key(request, previous_pickle)
             previous_cache_file = os.path.join(
                 self.cache_dir, f"{previous_cache_key}.json"
@@ -170,16 +170,14 @@ def _generate_cache_key(self, request: BackendRequest, pickle_path: str) -> str:
         print("Hash input: ", hash_input)
         return hashlib.md5(hash_input.encode()).hexdigest()
 
-    def _get_previous_pickle(self) -> Optional[str]:
-        print("Attempting previous pickle")
+    def _get_previous_pickles(self, num_pickles: int = 4) -> List[str]:
+        print(f"Attempting to get previous {num_pickles} pickles")
         _pickle_paths = glob.glob(f"{self.state.current_pickle_path}/../*")
-        pickle_paths = sorted([os.path.realpath(dir) for dir in _pickle_paths])
+        pickle_paths = sorted(
+            [os.path.realpath(dir) for dir in _pickle_paths], reverse=True
+        )
         print("Pickle paths: ", pickle_paths)
 
-        if len(pickle_paths) > 1:
-            previous_pickle_path = pickle_paths[-2]
-            print("Previous pickle: ", previous_pickle_path)
-            return previous_pickle_path
-
-        print("No previous pickle found")
-        return None
+        previous_pickles = pickle_paths[1 : num_pickles + 1]
+        print(f"Previous {len(previous_pickles)} pickles: ", previous_pickles)
+        return previous_pickles

backend/utils/waiting_for.py

@@ -22,7 +22,7 @@ def run(self):
                 file=self.file,
                 flush=True,
             )
-            time.sleep(0.1)
+            time.sleep(1)
 
     def start(self):
         self.is_running = True

src/page/price_shock.py

@@ -126,16 +126,21 @@ def price_shock_page():
     # Update query parameters
     st.query_params.update({"cov": cov, "oracle_distort": oracle_distort})
 
-    result = api(
-        "price-shock",
-        "usermap",
-        params={
-            "asset_group": cov,
-            "oracle_distortion": oracle_distort,
-            "n_scenarios": 5,
-        },
-        as_json=True,
-    )
+    try:
+        result = api(
+            "price-shock",
+            "usermap",
+            params={
+                "asset_group": cov,
+                "oracle_distortion": oracle_distort,
+                "n_scenarios": 5,
+            },
+            as_json=True,
+        )
+        # print("RESULT", result)
+    except Exception as e:
+        print("HIT AN EXCEPTION...", e)
+
     if "result" in result and result["result"] == "miss":
         st.write("Fetching data for the first time...")
         st.image(
@@ -144,8 +149,6 @@ def price_shock_page():
         st.write("Check again in one minute!")
         st.stop()
 
-    # st.write(result)
-
     fig = price_shock_plot(result, oracle_distort)
     st.plotly_chart(fig)