Moved update before file check and limited checks to once per hour.

intelmq/bots/parsers/shadowserver/_config.py

@@ -97,6 +97,7 @@ class __Container:
 __config = __Container()
 __config.var_state_path = VAR_STATE_PATH
 __config.schema_url = 'https://interchange.shadowserver.org/intelmq/v1/schema'
+__config.schema_check = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.check')
 __config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json')
 __config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test')
 __config.schema_active = __config.schema_file
@@ -330,6 +331,9 @@ def reload():
     """ reload the configuration if it has changed """
     mtime = 0.0
 
+    if __config.auto_update:
+        update_schema()
+
     if os.path.isfile(__config.schema_file):
         mtime = os.path.getmtime(__config.schema_file)
         if __config.schema_mtime == mtime:
@@ -338,9 +342,6 @@ def reload():
         if not __config.test_mode:
             raise ValueError("The schema file does not exist: %r.", __config.schema_file)
 
-    if __config.schema_mtime == 0.0 and mtime == 0.0 and __config.auto_update:
-        update_schema()
-
     __config.feedname_mapping.clear()
     __config.filename_mapping.clear()
     if os.path.isfile(__config.schema_active):
@@ -360,6 +361,14 @@ def reload():
 def update_schema():
     """ download the latest configuration """
 
+    # skip update if the last check was less than an hour ago
+    if os.path.isfile(__config.schema_check):
+        age = time.time() - os.path.getmtime(__config.schema_check)
+        if age < 3600:
+            return False
+    with open(__config.schema_check, "w") as myfile:
+        pass
+
     # download the schema to a temp file
     (th, tmp) = tempfile.mkstemp(dir=__config.var_state_path)
     __config.logger.info("Attempting to download schema from %r", __config.schema_url)