From fa16af5d1975fb331f2cd3f9205c4220ffcc5ed6 Mon Sep 17 00:00:00 2001
From: Michel Van den Bergh <michel.vandenbergh@uhasselt.be>
Date: Wed, 8 May 2024 04:17:27 +0000
Subject: [PATCH] Proof of concept inplementation of get_run via api.

/api/serialize_run hosted by the main instance can be invoked
by a secondary instance to receive a copy of the run from the
cache.

In this way there is never an issue that the run might not
be up to date.

Sadly I don't know how feasible this. A typical serialized run
is 7k and that is probably too much.

Note: currently /api/serialize_run is completely open. It should
be restricted to be only accessible from localhost.
---
 server/fishtest/__init__.py |  1 +
 server/fishtest/api.py      | 26 ++++++++++++++++++++++++--
 server/fishtest/rundb.py    | 12 +++++++++++-
 server/fishtest/util.py     | 17 +++++++++++++++++
 4 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/server/fishtest/__init__.py b/server/fishtest/__init__.py
index 028c3496b..33ad3cc50 100644
--- a/server/fishtest/__init__.py
+++ b/server/fishtest/__init__.py
@@ -142,6 +142,7 @@ def group_finder(username, request):
     config.add_route("api_get_elo", "/api/get_elo/{id}")
     config.add_route("api_actions", "/api/actions")
     config.add_route("api_calc_elo", "/api/calc_elo")
+    config.add_route("api_serialize_run", "/api/serialize_run/{id}")
 
     config.scan()
     return config.make_wsgi_app()
diff --git a/server/fishtest/api.py b/server/fishtest/api.py
index e8bc58529..e9e2712d3 100644
--- a/server/fishtest/api.py
+++ b/server/fishtest/api.py
@@ -7,7 +7,7 @@
 
 from fishtest.schemas import api_access_schema, api_schema, gzip_data
 from fishtest.stats.stat_util import SPRT_elo, get_elo
-from fishtest.util import worker_name
+from fishtest.util import serialize, worker_name
 from pyramid.httpexceptions import (
     HTTPBadRequest,
     HTTPFound,
@@ -90,7 +90,9 @@ def __init__(self, request):
 
     def handle_error(self, error, exception=HTTPBadRequest):
         if error != "":
-            full_url = self.request.route_url(self.request.matched_route.name)
+            full_url = self.request.route_url(
+                self.request.matched_route.name, **self.request.matchdict
+            )
             api = urlparse(full_url).path
             error = f"{api}: {error}"
             print(error, flush=True)
@@ -290,6 +292,26 @@ def finished_runs(self):
             finished[str(run["_id"])] = run
         return finished
 
+    @view_config(route_name="api_serialize_run")
+    def serialize_run(self):
+        self.__t0 = datetime.now(timezone.utc)
+        try:
+            run_id = self.request.matchdict["id"]
+        except Exception as e:
+            self.handle_error(str(e))
+        try:
+            run = self.request.rundb.get_run(run_id)
+        except Exception as e:
+            self.handle_error(str(e))
+        if run is None:
+            self.handle_error("Run does not exist")
+        try:
+            s = serialize(run)
+        except Exception as e:
+            self.handle_error(str(e))
+        result = {"run": s}
+        return self.add_time(result)
+
     @view_config(route_name="api_actions")
     def actions(self):
         try:
diff --git a/server/fishtest/rundb.py b/server/fishtest/rundb.py
index ba136bda7..312a8e04b 100644
--- a/server/fishtest/rundb.py
+++ b/server/fishtest/rundb.py
@@ -12,6 +12,7 @@
 from datetime import datetime, timedelta, timezone
 
 import fishtest.stats.stat_util
+import requests
 from bson.binary import Binary
 from bson.codec_options import CodecOptions
 from bson.objectid import ObjectId
@@ -22,6 +23,7 @@
 from fishtest.util import (
     GeneratorAsFileReader,
     crash_or_time,
+    deserialize,
     estimate_game_duration,
     format_bounds,
     format_results,
@@ -381,7 +383,15 @@ def get_run(self, r_id):
                     }
                 return run
         else:
-            return self.runs.find_one({"_id": ObjectId(r_id)})
+            result = requests.get(f"http://localhost/api/serialize_run/{r_id}")
+            try:
+                result.raise_for_status()
+                b = result.json()
+                run = deserialize(b["run"])
+            except Exception as e:
+                print(f"Failed to deserialize {r_id}: {str(e)}", flush=True)
+                return None
+            return run
 
     def start_timer(self):
         self.timer = threading.Timer(1.0, self.flush_buffers)
diff --git a/server/fishtest/util.py b/server/fishtest/util.py
index b7ea591af..a83501ef1 100644
--- a/server/fishtest/util.py
+++ b/server/fishtest/util.py
@@ -1,3 +1,5 @@
+import base64
+import gzip
 import hashlib
 import math
 import re
@@ -6,6 +8,7 @@
 from email.mime.text import MIMEText
 from functools import cache
 
+import bson
 import fishtest.stats.stat_util
 import numpy
 import scipy.stats
@@ -512,3 +515,17 @@ def get_hash(s):
     if h:
         return int(h.group(1))
     return 0
+
+
+def serialize(run):
+    b = bson.encode(run)
+    g = gzip.compress(b)
+    b64 = base64.b64encode(g).decode()
+    return b64
+
+
+def deserialize(run_b64):
+    g = base64.b64decode(run_b64)
+    b = gzip.decompress(g)
+    run = bson.decode(b)
+    return run