Ensure downloads are OK before cache write

The bundled request package doesn't verify the content downloaded is complete.
Hence verify that that content is correct before storing it to cache.
For the net perform a sha verify, for the zipball unzip first.

server/fishtest/api.py

@@ -34,7 +34,7 @@
 according to the route/URL mapping defined in `__init__.py`.
 """
 
-WORKER_VERSION = 241
+WORKER_VERSION = 242
 
 
 @exception_view_config(HTTPException)

worker/games.py

@@ -328,7 +328,9 @@ def download_net(remote, testing_dir, net, global_cache):
         url = remote + "/api/nn/" + net
         print("Downloading {}".format(net))
         content = requests_get(url, allow_redirects=True, timeout=HTTP_TIMEOUT).content
-        cache_write(global_cache, net, content)
+        hash = hashlib.sha256(content).hexdigest()
+        if hash[:12] == net[3:15]:
+            cache_write(global_cache, net, content)
     else:
         print("Using {} from global cache".format(net))
 
@@ -728,11 +730,16 @@ def setup_engine(
             item_url = github_api(repo_url) + "/zipball/" + sha
             print("Downloading {}".format(item_url))
             blob = requests_get(item_url).content
-            cache_write(global_cache, sha + ".zip", blob)
+            blob_needs_write = True
         else:
+            blob_needs_write = False
             print("Using {} from global cache".format(sha + ".zip"))
 
         file_list = unzip(blob, tmp_dir)
+        # once unzipped without error we can write as needed
+        if blob_needs_write:
+            cache_write(global_cache, sha + ".zip", blob)
+
         prefix = os.path.commonprefix([n.filename for n in file_list])
         os.chdir(tmp_dir / prefix / "src")
 

worker/sri.txt

@@ -1 +1 @@
-{"__version": 241, "updater.py": "Mg+pWOgGA0gSo2TuXuuLCWLzwGwH91rsW1W3ixg3jYauHQpRMtNdGnCfuD1GqOhV", "worker.py": "BMuQUpxZAKF0aP6ByTZY1r06MfPoIbdG2xraTrDQQRKgvhzJo6CKmeX2P8vX/QDm", "games.py": "9dFaa914vpqT7q4LLx2LlDdYwK6QFVX3h7+XRt18ATX0lt737rvFeBIiqakkttNC"}
+{"__version": 242, "updater.py": "Mg+pWOgGA0gSo2TuXuuLCWLzwGwH91rsW1W3ixg3jYauHQpRMtNdGnCfuD1GqOhV", "worker.py": "+66zNfqqaj7qSXNsX4QNznCFmcbNFl3U3r/afv8XcB6rz7d8q8GD9njQi8f8L/1J", "games.py": "nzbBgbzn479L3an/YlD4b08uk8mLt7htmwSDqaFVbS7s4UBUmgkPrNqvuLj9JRTd"}

worker/worker.py

@@ -68,7 +68,7 @@
 MIN_CLANG_MAJOR = 8
 MIN_CLANG_MINOR = 0
 
-WORKER_VERSION = 241
+WORKER_VERSION = 242
 FILE_LIST = ["updater.py", "worker.py", "games.py"]
 HTTP_TIMEOUT = 30.0
 INITIAL_RETRY_TIME = 15.0