Skip to content

Commit

Permalink
Moved update before file check and limited checks to once per hour.
Browse files Browse the repository at this point in the history
  • Loading branch information
elsif2 committed Mar 18, 2024
1 parent d58dba0 commit dc2afa7
Showing 1 changed file with 14 additions and 3 deletions.
17 changes: 14 additions & 3 deletions intelmq/bots/parsers/shadowserver/_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@
import binascii
import json
import tempfile
import time
from typing import Optional, Dict, Tuple, Any

import intelmq.lib.harmonization as harmonization
Expand All @@ -97,6 +98,7 @@ class __Container:
__config = __Container()
__config.var_state_path = VAR_STATE_PATH
__config.schema_url = 'https://interchange.shadowserver.org/intelmq/v1/schema'
__config.schema_check = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.check')
__config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json')
__config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test')
__config.schema_active = __config.schema_file
Expand Down Expand Up @@ -330,6 +332,9 @@ def reload():
""" reload the configuration if it has changed """
mtime = 0.0

if __config.auto_update:
update_schema()

if os.path.isfile(__config.schema_file):
mtime = os.path.getmtime(__config.schema_file)
if __config.schema_mtime == mtime:
Expand All @@ -338,9 +343,6 @@ def reload():
if not __config.test_mode:
raise ValueError("The schema file does not exist: %r.", __config.schema_file)

if __config.schema_mtime == 0.0 and mtime == 0.0 and __config.auto_update:
update_schema()

__config.feedname_mapping.clear()
__config.filename_mapping.clear()
if os.path.isfile(__config.schema_active):
Expand All @@ -360,6 +362,14 @@ def reload():
def update_schema():
""" download the latest configuration """

# skip update if the last check was less than an hour ago
if os.path.isfile(__config.schema_check):
age = time.time() - os.path.getmtime(__config.schema_check)
if age < 3600:
return False
with open(__config.schema_check, "w+") as myfile:
pass

# download the schema to a temp file
(th, tmp) = tempfile.mkstemp(dir=__config.var_state_path)
__config.logger.info("Attempting to download schema from %r", __config.schema_url)
Expand Down Expand Up @@ -415,5 +425,6 @@ def update_schema():
def prepare_update_schema_test(path):
""" Reconfigure internal settings to perform a schema update test. """
__config.var_state_path = path
__config.schema_check = os.path.join(path, 'shadowserver-schema.check')
__config.schema_file = os.path.join(path, 'shadowserver-schema.json')
return __config.schema_file

0 comments on commit dc2afa7

Please sign in to comment.