Some fixes in scavenge_dead_tasks

- Remove a race condition

- Do not require a cache entry to be clean before scavenging.

Under heavy load clean cache entries will be a rarity.

server/fishtest/rundb.py

@@ -517,13 +517,12 @@ def scavenge_dead_tasks(self):
         with self.run_cache_lock:
             for cache_entry in self.run_cache.values():
                 run = cache_entry["run"]
-                if not cache_entry["is_changed"] and not run["finished"]:
+                if not run["finished"]:
                     for task_id, task in enumerate(run["tasks"]):
                         if (
                             task["active"]
                             and task["last_updated"].timestamp() < now - 360
                         ):
-                            cache_entry["is_changed"] = True
                             dead_tasks.append((task_id, run))
         # We release the lock to avoid deadlock
         for task_id, run in dead_tasks:
@@ -541,6 +540,7 @@ def scavenge_dead_tasks(self):
                 task_id=task_id,
             )
             self.set_inactive_task(task_id, run)
+            self.buffer(run, False)
 
     def get_unfinished_runs_id(self):
         with self.run_cache_write_lock: