From 938430623ba50533b9276935e4af85968c33b9f1 Mon Sep 17 00:00:00 2001 From: Joost VandeVondele Date: Sun, 7 Jul 2024 09:21:58 +0200 Subject: [PATCH] Ensure downloads are OK before cache write The bundled request package doesn't verify the content downloaded is complete. Hence verify that that content is correct before storing it to cache. For the net perform a sha verify, for the zipball unzip first. --- server/fishtest/api.py | 2 +- worker/games.py | 11 +++++++++-- worker/sri.txt | 2 +- worker/worker.py | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/server/fishtest/api.py b/server/fishtest/api.py index 22755e5fa..6a403e512 100644 --- a/server/fishtest/api.py +++ b/server/fishtest/api.py @@ -34,7 +34,7 @@ according to the route/URL mapping defined in `__init__.py`. """ -WORKER_VERSION = 241 +WORKER_VERSION = 242 @exception_view_config(HTTPException) diff --git a/worker/games.py b/worker/games.py index 64463a3c5..72c3aacf8 100644 --- a/worker/games.py +++ b/worker/games.py @@ -328,7 +328,9 @@ def download_net(remote, testing_dir, net, global_cache): url = remote + "/api/nn/" + net print("Downloading {}".format(net)) content = requests_get(url, allow_redirects=True, timeout=HTTP_TIMEOUT).content - cache_write(global_cache, net, content) + hash = hashlib.sha256(content).hexdigest() + if hash[:12] == net[3:15]: + cache_write(global_cache, net, content) else: print("Using {} from global cache".format(net)) @@ -728,11 +730,16 @@ def setup_engine( item_url = github_api(repo_url) + "/zipball/" + sha print("Downloading {}".format(item_url)) blob = requests_get(item_url).content - cache_write(global_cache, sha + ".zip", blob) + blob_needs_write = True else: + blob_needs_write = False print("Using {} from global cache".format(sha + ".zip")) file_list = unzip(blob, tmp_dir) + # once unzipped without error we can write as needed + if blob_needs_write: + cache_write(global_cache, sha + ".zip", blob) + prefix = os.path.commonprefix([n.filename for n in file_list]) os.chdir(tmp_dir / prefix / "src") diff --git a/worker/sri.txt b/worker/sri.txt index ee2cad5b6..d83458829 100644 --- a/worker/sri.txt +++ b/worker/sri.txt @@ -1 +1 @@ -{"__version": 241, "updater.py": "Mg+pWOgGA0gSo2TuXuuLCWLzwGwH91rsW1W3ixg3jYauHQpRMtNdGnCfuD1GqOhV", "worker.py": "BMuQUpxZAKF0aP6ByTZY1r06MfPoIbdG2xraTrDQQRKgvhzJo6CKmeX2P8vX/QDm", "games.py": "9dFaa914vpqT7q4LLx2LlDdYwK6QFVX3h7+XRt18ATX0lt737rvFeBIiqakkttNC"} +{"__version": 242, "updater.py": "Mg+pWOgGA0gSo2TuXuuLCWLzwGwH91rsW1W3ixg3jYauHQpRMtNdGnCfuD1GqOhV", "worker.py": "+66zNfqqaj7qSXNsX4QNznCFmcbNFl3U3r/afv8XcB6rz7d8q8GD9njQi8f8L/1J", "games.py": "nzbBgbzn479L3an/YlD4b08uk8mLt7htmwSDqaFVbS7s4UBUmgkPrNqvuLj9JRTd"} diff --git a/worker/worker.py b/worker/worker.py index bfc40c95d..9b861b13e 100644 --- a/worker/worker.py +++ b/worker/worker.py @@ -68,7 +68,7 @@ MIN_CLANG_MAJOR = 8 MIN_CLANG_MINOR = 0 -WORKER_VERSION = 241 +WORKER_VERSION = 242 FILE_LIST = ["updater.py", "worker.py", "games.py"] HTTP_TIMEOUT = 30.0 INITIAL_RETRY_TIME = 15.0