From c13738919d31486fc463108cf6f55de39748fec7 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 18 Mar 2024 23:51:14 +0000 Subject: [PATCH] Moved update before file check and limited checks to once per hour. --- intelmq/bots/parsers/shadowserver/_config.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 6f3be4c14..2ef11a91c 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -97,6 +97,7 @@ class __Container: __config = __Container() __config.var_state_path = VAR_STATE_PATH __config.schema_url = 'https://interchange.shadowserver.org/intelmq/v1/schema' +__config.schema_check = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.check') __config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') __config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') __config.schema_active = __config.schema_file @@ -330,6 +331,9 @@ def reload(): """ reload the configuration if it has changed """ mtime = 0.0 + if __config.auto_update: + update_schema() + if os.path.isfile(__config.schema_file): mtime = os.path.getmtime(__config.schema_file) if __config.schema_mtime == mtime: @@ -338,9 +342,6 @@ def reload(): if not __config.test_mode: raise ValueError("The schema file does not exist: %r.", __config.schema_file) - if __config.schema_mtime == 0.0 and mtime == 0.0 and __config.auto_update: - update_schema() - __config.feedname_mapping.clear() __config.filename_mapping.clear() if os.path.isfile(__config.schema_active): @@ -360,6 +361,14 @@ def reload(): def update_schema(): """ download the latest configuration """ + # skip update if the last check was less than an hour ago + if os.path.isfile(__config.schema_check): + age = time.time() - os.path.getmtime(__config.schema_check) + if age < 3600: + return False + with open(__config.schema_check, "w+") as myfile: + pass + # download the schema to a temp file (th, tmp) = tempfile.mkstemp(dir=__config.var_state_path) __config.logger.info("Attempting to download schema from %r", __config.schema_url)