Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Initial fetch of shadowserver-schema.json fails even in auto_update mode #2482

Merged
merged 1 commit into from
Apr 4, 2024
Merged

Initial fetch of shadowserver-schema.json fails even in auto_update mode #2482

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 14 additions & 3 deletions intelmq/bots/parsers/shadowserver/_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@
import binascii
import json
import tempfile
import time
from typing import Optional, Dict, Tuple, Any

import intelmq.lib.harmonization as harmonization
Expand All @@ -97,6 +98,7 @@ class __Container:
__config = __Container()
__config.var_state_path = VAR_STATE_PATH
__config.schema_url = 'https://interchange.shadowserver.org/intelmq/v1/schema'
__config.schema_check = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.check')
__config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json')
__config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test')
__config.schema_active = __config.schema_file
Expand Down Expand Up @@ -330,6 +332,9 @@ def reload():
""" reload the configuration if it has changed """
mtime = 0.0

if __config.auto_update:
update_schema()

if os.path.isfile(__config.schema_file):
mtime = os.path.getmtime(__config.schema_file)
if __config.schema_mtime == mtime:
Expand All @@ -338,9 +343,6 @@ def reload():
if not __config.test_mode:
raise ValueError("The schema file does not exist: %r.", __config.schema_file)

if __config.schema_mtime == 0.0 and mtime == 0.0 and __config.auto_update:
update_schema()

__config.feedname_mapping.clear()
__config.filename_mapping.clear()
if os.path.isfile(__config.schema_active):
Expand All @@ -360,6 +362,14 @@ def reload():
def update_schema():
""" download the latest configuration """

# skip update if the last check was less than an hour ago
if os.path.isfile(__config.schema_check):
age = time.time() - os.path.getmtime(__config.schema_check)
if age < 3600:
return False
with open(__config.schema_check, "w+") as myfile:
pass

# download the schema to a temp file
(th, tmp) = tempfile.mkstemp(dir=__config.var_state_path)
__config.logger.info("Attempting to download schema from %r", __config.schema_url)
Expand Down Expand Up @@ -415,5 +425,6 @@ def update_schema():
def prepare_update_schema_test(path):
""" Reconfigure internal settings to perform a schema update test. """
__config.var_state_path = path
__config.schema_check = os.path.join(path, 'shadowserver-schema.check')
__config.schema_file = os.path.join(path, 'shadowserver-schema.json')
return __config.schema_file
Loading