Documentation and style updates.

certtools · May 28, 2023 · 41a6c63 · 41a6c63
1 parent 819983b
commit 41a6c63
Show file tree

Hide file tree

Showing 10 changed files with 151 additions and 117 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -24,10 +24,13 @@ CHANGELOG
 ### Bots
 
 #### Collectors
+- `intelmq.bots.collectors.shadowserver.collector_reports_api`:
+  - The 'json' option is no longer supported as the 'csv' option provides better performance.  Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372)
 
 #### Parsers
 - `intelmq.bots.parsers.shadowserver._config`:
   - Reset detected `feedname` at shutdown to re-detect the feedname on reloads (PR#2361 by @elsif2, fixes #2360).
+  - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases.  Please see intelmq/bots/parsers/shadowserver/README.md for details. (PR#2372)
 - `intelmq.bots.parsers.shadowserver._config`:
   - Added 'IPv6-Vulnerable-Exchange' alias and 'Accessible-WS-Discovery-Service' report. (PR#2338)
   - Removed unused 'p0f_genre' and 'p0f_detail' from the 'DNS-Open-Resolvers' report. (PR#2338)

diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py
@@ -34,7 +34,7 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin):
             A list of strings or a comma-separated list of the mailing lists you want to process.
         types (list):
             A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined').
-        file_format (str): File format to download ('csv').  The 'json' option is not longer supported.
+        file_format (str): File format to download ('csv').  The 'json' option is no longer supported.
     """
 
     country = None

diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md
@@ -7,10 +7,45 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser
 
 Please contact [email protected] with any issues or concerns.
 
-The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1.
+The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema.
 
-For environments that have internet connectivity the `update_schema.py` script can be called from a cron job to obtain the latest revision.
+For environments that have internet connectivity the `update_schema.py` script should be called from a cron job to obtain the latest revision.
+The parser will attempt to download a schema update on startup unless INTELMQ_SKIP_INTERNET is set.
 
 For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory.
 
 The parser will automatically reload the configuration when the file changes.
+
+
+## Sample configuration:
+
+```
+shadowserver-collector:
+  description: Our bot responsible for getting reports from Shadowserver
+  enabled: true
+  group: Collector
+  module: intelmq.bots.collectors.shadowserver.collector_reports_api
+  name: Shadowserver_Collector
+  parameters:
+    destination_queues:
+      _default: [shadowserver-parser-queue]
+    file_format: csv
+    api_key: "$API_KEY_received_from_the_shadowserver_foundation"
+    secret: "$SECRET_received_from_the_shadowserver_foundation"
+  run_mode: continuous
+```
+
+```
+shadowserver-parser:
+  bot_id: shadowserver-parser
+  name: Shadowserver Parser
+  enabled: true
+  group: Parser
+  groupname: parsers
+  module: intelmq.bots.parsers.shadowserver.parser
+  parameters:
+    destination_queues:
+      _default: [file-output-queue]
+  run_mode: continuous
+```
+
diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py
@@ -88,15 +88,18 @@
 
 import intelmq.lib.harmonization as harmonization
 
+
 class __Container:
     pass
 
+
 __config = __Container()
 __config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json')
 __config.schema_mtime = 0.0
 __config.feedname_mapping = {}
 __config.filename_mapping = {}
 
+
 def set_logger(logger):
     """ Sets the logger instance. """
     __config.logger = logger
@@ -254,27 +257,28 @@ def scan_exchange_identifier(field):
         return 'exchange-server-webshell'
     return 'vulnerable-exchange-server'
 
+
 functions = {
-        'add_UTC_to_timestamp': add_UTC_to_timestamp,
-        'convert_bool': convert_bool,
-        'validate_to_none': validate_to_none,
-        'convert_int': convert_int,
-        'convert_float': convert_float,
-        'convert_http_host_and_url': convert_http_host_and_url,
-        'invalidate_zero': invalidate_zero,
-        'validate_ip': validate_ip,
-        'validate_network': validate_network,
-        'validate_fqdn': validate_fqdn,
-        'convert_date': convert_date,
-        'convert_date_utc': convert_date_utc,
-        'force_base64': force_base64,
-        'scan_exchange_taxonomy': scan_exchange_taxonomy,
-        'scan_exchange_type': scan_exchange_type,
-        'scan_exchange_identifier': scan_exchange_identifier,
-        }
-
-
-def reload ():
+    'add_UTC_to_timestamp': add_UTC_to_timestamp,
+    'convert_bool': convert_bool,
+    'validate_to_none': validate_to_none,
+    'convert_int': convert_int,
+    'convert_float': convert_float,
+    'convert_http_host_and_url': convert_http_host_and_url,
+    'invalidate_zero': invalidate_zero,
+    'validate_ip': validate_ip,
+    'validate_network': validate_network,
+    'validate_fqdn': validate_fqdn,
+    'convert_date': convert_date,
+    'convert_date_utc': convert_date_utc,
+    'force_base64': force_base64,
+    'scan_exchange_taxonomy': scan_exchange_taxonomy,
+    'scan_exchange_type': scan_exchange_type,
+    'scan_exchange_identifier': scan_exchange_identifier,
+}
+
+
+def reload():
     """ reload the configuration if it has changed """
     mtime = 0.0
 
@@ -291,7 +295,7 @@ def reload ():
 
     __config.feedname_mapping.clear()
     __config.filename_mapping.clear()
-    for schema_file in [ __config.schema_file, ".".join([__config.schema_file, 'test']) ]:
+    for schema_file in [__config.schema_file, ".".join([__config.schema_file, 'test'])]:
         if os.path.isfile(schema_file):
             with open(schema_file) as fh:
                 schema = json.load(fh)
@@ -305,13 +309,14 @@ def reload ():
                     __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report])
     __config.schema_mtime = mtime
 
-def update_schema ():
+
+def update_schema():
     """ download the latest configuration """
     if os.environ.get('INTELMQ_SKIP_INTERNET'):
         return None
 
     (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__))
-    url = 'https://interchange.shadowserver.org/intelmq/v1'
+    url = 'https://interchange.shadowserver.org/intelmq/v1/schema'
     try:
         urllib.request.urlretrieve(url, tmp)
     except:
@@ -329,7 +334,6 @@ def update_schema ():
         raise ValueError("Failed to validate %r" % tmp)
 
     if os.path.exists(__config.schema_file):
-        old_version = ''
         try:
             with open(__config.schema_file) as fh:
                 schema = json.load(fh)

diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py
@@ -51,7 +51,7 @@ def init(self):
         try:
             config.update_schema()
         except Exception as e:
-            logger.warning(f"Schema update failed: {e}.")
+            self.logger.warning("Schema update failed: %s." % e)
         if self.feedname is not None:
             self._sparser_config = config.get_feed_by_feedname(self.feedname)
             if self._sparser_config:

diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py
@@ -24,12 +24,12 @@
            "__type": "Report",
            "time.observation": "2015-01-01T00:00:00+00:00",
            "extra.file_name": "2019-01-01-some_string-test-test.csv",
-}
+           }
 REPORT4 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'),
            "__type": "Report",
            "time.observation": "2015-01-01T00:00:00+00:00",
            "extra.file_name": "2020.wrong-filename.csv",
-}
+           }
 
 
 class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase):

diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py
@@ -52,6 +52,5 @@ def test_changed_feed(self):
         self.run_bot(iterations=2)
 
 
-
 if __name__ == '__main__':  # pragma: no cover
     unittest.main()
diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py
@@ -22,85 +22,78 @@
                   "time.observation": "2015-01-01T00:00:00+00:00",
                   "extra.file_name": "2019-01-01-test_smb-test-geo.csv",
                   }
-EVENTS = [
-{
-   '__type' : 'Event',
-   'classification.identifier' : 'test-smb',
-   'classification.taxonomy' : 'vulnerable',
-   'classification.type' : 'vulnerable-system',
-   'extra.smb_implant' : False,
-   'extra.smb_major_number' : '2',
-   'extra.smb_minor_number' : '1',
-   'extra.smb_version_string' : 'SMB 2.1',
-   'extra.smbv1_support' : 'N',
-   'extra.tag' : 'smb',
-   'feed.name' : 'Test-Accessible-SMB',
-   'protocol.application' : 'smb',
-   'protocol.transport' : 'tcp',
-   'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])),
-   'source.asn' : 64512,
-   'source.geolocation.cc' : 'ZZ',
-   'source.geolocation.city' : 'City',
-   'source.geolocation.region' : 'Region',
-   'source.ip' : '192.168.0.1',
-   'source.port' : 445,
-   'source.reverse_dns' : 'node01.example.com',
-   'time.observation' : '2015-01-01T00:00:00+00:00',
-   'time.source' : '2010-02-10T00:00:00+00:00'
-},
-
-{
-   '__type' : 'Event',
-   'classification.identifier' : 'test-smb',
-   'classification.taxonomy' : 'vulnerable',
-   'classification.type' : 'vulnerable-system',
-   'extra.smb_implant' : False,
-   'extra.smb_major_number' : '2',
-   'extra.smb_minor_number' : '1',
-   'extra.smb_version_string' : 'SMB 2.1',
-   'extra.smbv1_support' : 'N',
-   'extra.tag' : 'smb',
-   'feed.name' : 'Test-Accessible-SMB',
-   'protocol.application' : 'smb',
-   'protocol.transport' : 'tcp',
-   'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])),
-   'source.asn' : 64512,
-   'source.geolocation.cc' : 'ZZ',
-   'source.geolocation.city' : 'City',
-   'source.geolocation.region' : 'Region',
-   'source.ip' : '192.168.0.2',
-   'source.port' : 445,
-   'source.reverse_dns' : 'node02.example.com',
-   'time.observation' : '2015-01-01T00:00:00+00:00',
-   'time.source' : '2010-02-10T00:00:01+00:00'
-},
-
-{
-   '__type' : 'Event',
-   'classification.identifier' : 'test-smb',
-   'classification.taxonomy' : 'vulnerable',
-   'classification.type' : 'vulnerable-system',
-   'extra.smb_implant' : False,
-   'extra.smb_major_number' : '2',
-   'extra.smb_minor_number' : '1',
-   'extra.smb_version_string' : 'SMB 2.1',
-   'extra.smbv1_support' : 'N',
-   'extra.tag' : 'smb',
-   'feed.name' : 'Test-Accessible-SMB',
-   'protocol.application' : 'smb',
-   'protocol.transport' : 'tcp',
-   'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])),
-   'source.asn' : 64512,
-   'source.geolocation.cc' : 'ZZ',
-   'source.geolocation.city' : 'City',
-   'source.geolocation.region' : 'Region',
-   'source.ip' : '192.168.0.3',
-   'source.port' : 445,
-   'source.reverse_dns' : 'node03.example.com',
-   'time.observation' : '2015-01-01T00:00:00+00:00',
-   'time.source' : '2010-02-10T00:00:02+00:00'
-}
-          ]
+EVENTS = [{'__type': 'Event',
+           'classification.identifier': 'test-smb',
+           'classification.taxonomy': 'vulnerable',
+           'classification.type': 'vulnerable-system',
+           'extra.smb_implant': False,
+           'extra.smb_major_number': '2',
+           'extra.smb_minor_number': '1',
+           'extra.smb_version_string': 'SMB 2.1',
+           'extra.smbv1_support': 'N',
+           'extra.tag': 'smb',
+           'feed.name': 'Test-Accessible-SMB',
+           'protocol.application': 'smb',
+           'protocol.transport': 'tcp',
+           'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])),
+           'source.asn': 64512,
+           'source.geolocation.cc': 'ZZ',
+           'source.geolocation.city': 'City',
+           'source.geolocation.region': 'Region',
+           'source.ip': '192.168.0.1',
+           'source.port': 445,
+           'source.reverse_dns': 'node01.example.com',
+           'time.observation': '2015-01-01T00:00:00+00:00',
+           'time.source': '2010-02-10T00:00:00+00:00'
+           },
+          {'__type': 'Event',
+           'classification.identifier': 'test-smb',
+           'classification.taxonomy': 'vulnerable',
+           'classification.type': 'vulnerable-system',
+           'extra.smb_implant': False,
+           'extra.smb_major_number': '2',
+           'extra.smb_minor_number': '1',
+           'extra.smb_version_string': 'SMB 2.1',
+           'extra.smbv1_support': 'N',
+           'extra.tag': 'smb',
+           'feed.name': 'Test-Accessible-SMB',
+           'protocol.application': 'smb',
+           'protocol.transport': 'tcp',
+           'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])),
+           'source.asn': 64512,
+           'source.geolocation.cc': 'ZZ',
+           'source.geolocation.city': 'City',
+           'source.geolocation.region': 'Region',
+           'source.ip': '192.168.0.2',
+           'source.port': 445,
+           'source.reverse_dns': 'node02.example.com',
+           'time.observation': '2015-01-01T00:00:00+00:00',
+           'time.source': '2010-02-10T00:00:01+00:00'
+           },
+          {'__type': 'Event',
+           'classification.identifier': 'test-smb',
+           'classification.taxonomy': 'vulnerable',
+           'classification.type': 'vulnerable-system',
+           'extra.smb_implant': False,
+           'extra.smb_major_number': '2',
+           'extra.smb_minor_number': '1',
+           'extra.smb_version_string': 'SMB 2.1',
+           'extra.smbv1_support': 'N',
+           'extra.tag': 'smb',
+           'feed.name': 'Test-Accessible-SMB',
+           'protocol.application': 'smb',
+           'protocol.transport': 'tcp',
+           'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])),
+           'source.asn': 64512,
+           'source.geolocation.cc': 'ZZ',
+           'source.geolocation.city': 'City',
+           'source.geolocation.region': 'Region',
+           'source.ip': '192.168.0.3',
+           'source.port': 445,
+           'source.reverse_dns': 'node03.example.com',
+           'time.observation': '2015-01-01T00:00:00+00:00',
+           'time.source': '2010-02-10T00:00:02+00:00'
+           }]
 
 
 class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase):

diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py
@@ -16,11 +16,11 @@
     EXAMPLE_LINES = handle.read().splitlines()[:2]
 
 FIRST_REPORT = {'feed.name': 'Test-Accessible-SMB',
-                  "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)),
-                  "__type": "Report",
-                  "time.observation": "2019-03-25T00:00:00+00:00",
-                  "extra.file_name": "2019-03-25-test_smb-test-test.csv",
-                  }
+                "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)),
+                "__type": "Report",
+                "time.observation": "2019-03-25T00:00:00+00:00",
+                "extra.file_name": "2019-03-25-test_smb-test-test.csv",
+                }
 with open(os.path.join(os.path.dirname(__file__), 'testdata/test_telnet.csv')) as handle:
     EXAMPLE_LINES = handle.read().splitlines()[:2]
 

diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py
@@ -42,7 +42,7 @@
            "source.reverse_dns": "example.local",
            "time.observation": "2015-01-01T00:00:00+00:00",
            "time.source": "2019-09-04T12:27:34+00:00"
-          },
+           },
           {'__type': 'Event',
            'feed.name': 'Test-Accessible-Telnet',
            "classification.identifier": "test-telnet",
@@ -63,7 +63,7 @@
            "source.reverse_dns": "example.local",
            "time.observation": "2015-01-01T00:00:00+00:00",
            "time.source": "2019-09-04T12:27:40+00:00"
-          }]
+           }]
 
 
 class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase):