Merge pull request #2352 from monoidic/develop

ENH: cymru expert: prevent extra lookups, handle invalid asnames

CHANGELOG.md

@@ -10,6 +10,8 @@ CHANGELOG
 ------------------
 
 ### Core
+- `intelmq.lib.utils`:
+  - `resolve_dns`: Deprecate dnspython versions pre-2.0.0 and disable search domains (PR#2352)
 - Fixed not resetting destination path statistics in the stats cache after restarting bot (Fixes [#2331](https://github.com/certtools/intelmq/issues/2331))
 - Force flushing statistics if bot will sleep longer than flushing delay (Fixes [#2336](https://github.com/certtools/intelmq/issues/2336))
 
@@ -32,6 +34,9 @@ CHANGELOG
 #### Experts
 - `intelmq.bots.experts.sieve`:
   - Allow empty lists in sieve rule files (PR#2341 by Mikk Margus Möll).
+- `intelmq.bots.experts.cymru_whois`:
+  - Ignore AS names with unexpected unicode characters (PR#2352, fixes #2132)
+  - Avoid extraneous search domain-based queries on NXDOMAIN result (PR#2352)
 
 #### Outputs
 - `intelmq.bots.output.cif3.output`: Added (PR#2244 by Michael Davis).

debian/control

@@ -33,7 +33,7 @@ Depends: bash-completion,
          cron,
          jq,
          python3-dateutil (>= 2.5),
-         python3-dnspython (>= 1.11.1),
+         python3-dnspython (>= 2.0.0),
          python3-openssl,
          python3-psutil (>= 1.2.1),
          python3-redis (>= 2.10),

docs/requirements.txt

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 Sphinx>=3.2.1
-dnspython>=1.11.1
+dnspython>=2.0.0
 psutil>=1.2.1
 python-dateutil>=2.5
 python-termstyle>=0.1.10

intelmq/bots/experts/cymru_whois/_lib.py

@@ -168,6 +168,8 @@ def __asn_query_parse(text):
         if items[4]:
             # unicode characters need to be decoded explicitly
             # with the help of https://stackoverflow.com/questions/60890590/
-            result['as_name'] = items[4].encode('latin1').decode('utf8')
+            as_name = items[4].encode('latin1', errors='ignore').decode('utf8')
+            if as_name:
+                result['as_name'] = as_name
 
         return result

intelmq/lib/utils.py

@@ -933,9 +933,6 @@ def resolve_dns(*args, **kwargs) -> dns.resolver.Answer:
 
     Parameters:
         see: https://dnspython.readthedocs.io/en/stable/resolver-class.html#dns.resolver.Resolver.resolve
-        The `search` parameter is always set to True for compatibility with dnspython version 1.
 
     """
-    if dns.version.MAJOR < 2:
-        return dns.resolver.query(*args, **kwargs)
-    return dns.resolver.resolve(*args, **kwargs, search=True)
+    return dns.resolver.resolve(*args, **kwargs)

intelmq/tests/bots/experts/cymru_whois/test_expert.py

@@ -55,6 +55,11 @@
                   "source.as_name": "ACONET ACOnet Backbone, AT",
                   "time.observation": "2015-01-01T00:00:00+00:00",
                   }
+UNEXPECTED_UNICODE = {"__type": "Event",
+                      "source.asn": 266522}
+UNKNOWN_IP = {"__type": "Event",
+              "source.ip": "255.255.255.210",
+              }
 
 
 @test.skip_redis()
@@ -106,6 +111,10 @@ def test_overwrite(self):
         self.run_bot(parameters={'overwrite': False})
         self.assertMessageEqual(0, OVERWRITE_OUT)
 
+    def test_unexpected_unicode(self):
+        self.input_message = UNEXPECTED_UNICODE.copy()
+        self.run_bot()
+        self.assertMessageEqual(0, UNEXPECTED_UNICODE)
 
 if __name__ == '__main__':  # pragma: no cover
     unittest.main()

intelmq/tests/lib/test_utils.py

@@ -368,20 +368,6 @@ def test_load_configuration_yaml_invalid(self):
         with self.assertRaises(ScannerError):
             utils.load_configuration(filename)
 
-    @unittest.mock.patch.object(utils.dns.version, "MAJOR", 1)
-    def test_resolve_dns_supports_dnspython_v1(self):
-        with unittest.mock.patch.object(utils.dns.resolver, "query") as query_mocks:
-            utils.resolve_dns("example.com", "any", other="parameter")
-
-        query_mocks.assert_called_once_with("example.com", "any", other="parameter")
-
-    @unittest.mock.patch.object(utils.dns.version, "MAJOR", 2)
-    def test_resolve_dns_supports_dnspython_v2(self):
-        with unittest.mock.patch.object(utils.dns.resolver, "resolve", create=True) as resolve_mocks:
-            utils.resolve_dns("example.com", "any", other="parameter")
-
-        resolve_mocks.assert_called_once_with("example.com", "any", other="parameter", search=True)
-
     @skip_internet()
     def test_resolve_dns_returns_answer(self):
         answer = utils.resolve_dns("example.com")

setup.py

@@ -10,7 +10,7 @@
 from setuptools import find_packages, setup
 
 REQUIRES = [
-    'dnspython>=1.11.1',
+    'dnspython>=2.0.0',
     'psutil>=1.2.1',
     'python-dateutil>=2.5',
     'python-termstyle>=0.1.10',