Update memory estimates

after the various net updates the estimated memory need was no longer accurate
also add the estimated size of cutechess-cli with the larger book

server/fishtest/api.py

@@ -32,7 +32,7 @@
 This depends on how frequently the main instance flushes its `run_cache`.
 """
 
-WORKER_VERSION = 232
+WORKER_VERSION = 233
 
 
 def validate_request(request):

server/fishtest/rundb.py

@@ -912,17 +912,17 @@ def priority(run):  # lower is better
 
             # We check if the worker has reserved enough memory
             need_tt = 0
-            need_base = 0
-
             need_tt += get_hash(run["args"]["new_options"])
             need_tt += get_hash(run["args"]["base_options"])
             need_tt *= max_threads // run["args"]["threads"]
-            # estime another 10MB per process, 30MB per thread, and 80MB for net as a base memory need besides hash
+            # Needed for cutechess-cli with the fairly large UHO_Lichess_4852_v1.epd opening book
+            need_base = 60
+            # estime another 10MB per process, 16MB per thread, and 132+6MB for large and small net
             # Note that changes here need the corresponding worker change to STC_memory, which limits concurrency
-            need_base = (
+            need_base += (
                 2
                 * (max_threads // run["args"]["threads"])
-                * (10 + 80 + 30 * run["args"]["threads"])
+                * (10 + 138 + 16 * run["args"]["threads"])
             )
 
             if need_base + need_tt > max_memory:
@@ -1546,7 +1546,9 @@ def generate_spsa(self, run):
                 {
                     "name": param["name"],
                     "value": self.spsa_param_clip(param, c * flip),
-                    "R": param["a"] / (spsa["A"] + iter_local) ** spsa["alpha"] / c**2,
+                    "R": param["a"]
+                    / (spsa["A"] + iter_local) ** spsa["alpha"]
+                    / c**2,
                     "c": c,
                     "flip": flip,
                 }

worker/sri.txt

@@ -1 +1 @@
-{"__version": 232, "updater.py": "Mg+pWOgGA0gSo2TuXuuLCWLzwGwH91rsW1W3ixg3jYauHQpRMtNdGnCfuD1GqOhV", "worker.py": "DRLqEwLvuSBzNt6k3MYtlC87bp6uwdUZ/O7Xs3/d+gDox0jTOT6SO/FNWzsXVs51", "games.py": "PrNaUxAFP0wPV/jCLivA0g9mcO1vq1Og1nN3vuPa408qcKoAYuza5ZHgDCZitkQG"}
+{"__version": 233, "updater.py": "Mg+pWOgGA0gSo2TuXuuLCWLzwGwH91rsW1W3ixg3jYauHQpRMtNdGnCfuD1GqOhV", "worker.py": "8aBsMobAmObrJ38c5+8RhV6T/UP0E2j9Y8m3dIzsFi8PAwWFi7RQx9+l6bMTNblS", "games.py": "PrNaUxAFP0wPV/jCLivA0g9mcO1vq1Og1nN3vuPa408qcKoAYuza5ZHgDCZitkQG"}

worker/worker.py

@@ -55,7 +55,7 @@
 # Several packages are called "expression".
 # So we make sure to use the locally installed one.
 
-WORKER_VERSION = 232
+WORKER_VERSION = 233
 FILE_LIST = ["updater.py", "worker.py", "games.py"]
 HTTP_TIMEOUT = 30.0
 INITIAL_RETRY_TIME = 15.0
@@ -795,10 +795,11 @@ def my_error(e):
         options.port = 443
 
     # Limit concurrency so that at least STC tests can run with the evailable memory
-    # The memory need per engine is 16 for the TT Hash, 10 for the process 80 for the net and 30 per thread
+    # The memory need per engine is 16 for the TT Hash, 10 for the process 138 for the net and 16 per thread
+    # 60 is the need for cutechess-cli
     # These numbers need to be up-to-date with the server values
-    STC_memory = 2 * (16 + 10 + 80 + 30)
-    max_concurrency = int(options.max_memory / STC_memory)
+    STC_memory = 2 * (16 + 10 + 138 + 16)
+    max_concurrency = int((options.max_memory - 60) / STC_memory)
     if max_concurrency < 1:
         print(
             "You need to reserve at least {} MiB to run the worker!".format(STC_memory)