From abbc5d393752212a0532489aa33a348c5093273a Mon Sep 17 00:00:00 2001 From: azazelm3dj3d <56496067+azazelm3dj3d@users.noreply.github.com> Date: Tue, 3 Oct 2023 13:02:32 -0500 Subject: [PATCH 1/3] Added BugSnag error reporting, initial pass --- threatingestor/__init__.py | 14 +++++++- threatingestor/bugsnagmonitor.py | 20 +++++++++++ threatingestor/config.py | 5 +++ threatingestor/operators/twitter.py | 54 ----------------------------- 4 files changed, 38 insertions(+), 55 deletions(-) create mode 100644 threatingestor/bugsnagmonitor.py delete mode 100644 threatingestor/operators/twitter.py diff --git a/threatingestor/__init__.py b/threatingestor/__init__.py index 67a1e6c..c8f8c22 100644 --- a/threatingestor/__init__.py +++ b/threatingestor/__init__.py @@ -17,6 +17,8 @@ import threatingestor.exceptions import threatingestor.whitelist +from threatingestor.bugsnagmonitor import configure_bugsnag, send_notification + class Ingestor: """ThreatIngestor main work logic. @@ -35,6 +37,7 @@ def __init__(self, config_file): # Configure logging with optional notifiers. logger.configure(**self.config.logging()) + try: logger.level("NOTIFY", no=35, color="", icon="\U0001F514") except TypeError: @@ -60,6 +63,12 @@ def __init__(self, config_file): except TypeError: logger.exception("Couldn't initialize statsd client; bad config?") sys.exit(1) + + # Configure BugSnag + for service in self.config.error_reporting(): + if service['name'] == "bugsnag": + configure_bugsnag(service['api_key']) + break # Load state DB. try: @@ -68,6 +77,7 @@ def __init__(self, config_file): except (OSError, IOError, threatingestor.exceptions.IngestorError): # Error loading state DB. logger.exception("Error reading state database") + send_notification("Error reading state database") sys.exit(1) # Instantiate plugins. @@ -84,8 +94,8 @@ def __init__(self, config_file): self.whitelist = threatingestor.whitelist.Whitelist(self.config.whitelists()) except (TypeError, ConnectionError, threatingestor.exceptions.PluginError): - logger.warning("Twitter config format has recently changed. See https://github.com/InQuest/ThreatIngestor/releases/tag/v1.0.0b5") logger.exception("Error initializing plugins") + send_notification("Error initializing plugins") sys.exit(1) def _is_whitelisted(self, artifact) -> bool: @@ -122,6 +132,7 @@ def run_once(self): except Exception: self.statsd.incr(f'error.source.{source}') logger.exception(f"Unknown error in source '{source}'") + send_notification(f"Unknown error in source '{source}'") continue # Save the source state. @@ -144,6 +155,7 @@ def run_once(self): except Exception: self.statsd.incr(f'error.operator.{operator}') logger.exception(f"Unknown error in operator '{operator}'") + send_notification(f"Unknown error in operator '{operator}'") continue # Record stats and update the summary. diff --git a/threatingestor/bugsnagmonitor.py b/threatingestor/bugsnagmonitor.py new file mode 100644 index 0000000..216f40f --- /dev/null +++ b/threatingestor/bugsnagmonitor.py @@ -0,0 +1,20 @@ +import bugsnag + +def configure_bugsnag(api_key=None) -> bool: + """ + Configure BugSnag API communication in `config.yml`. + """ + + if api_key: + bugsnag.configure(api_key=api_key) + return True + + return False + +def send_notification(msg=None, metadata=None) -> None: + """ + Monitor your code with BugSnag + + You can include a additional information with the `metadata` paramater. + """ + bugsnag.notify(Exception(msg), metadata={"ThreatIngestor" : metadata}) diff --git a/threatingestor/config.py b/threatingestor/config.py index 5a96566..c8bc9d7 100644 --- a/threatingestor/config.py +++ b/threatingestor/config.py @@ -77,6 +77,11 @@ def logging(self): return self.config.get('logging', {}) + def error_reporting(self): + """Returns error_reporting config dictionary.""" + return self.config['error_reporting'] + + def credentials(self, credential_name): """Return a dictionary with the specified credentials.""" for credential in self.config['credentials']: diff --git a/threatingestor/operators/twitter.py b/threatingestor/operators/twitter.py deleted file mode 100644 index 19a83c1..0000000 --- a/threatingestor/operators/twitter.py +++ /dev/null @@ -1,54 +0,0 @@ -from __future__ import absolute_import - -import re -import twitter -from loguru import logger - -from threatingestor.operators import Operator -import threatingestor.artifacts -import threatingestor.exceptions - -TWEET_URL = re.compile(r'https://twitter\.com/\w{1,15}/status/\d+') - -class Plugin(Operator): - """Operator for Twitter.""" - def __init__(self, api_key, api_secret_key, access_token, access_token_secret, status, **kwargs): - self.api = twitter.Twitter(auth=twitter.OAuth(access_token, access_token_secret, api_key, api_secret_key)) - self.status = status - - # Validate status, for better error handling. - if not isinstance(self.status, str): - raise threatingestor.exceptions.IngestorError(f"Invalid 'status' config: {self.status}") - - super(Plugin, self).__init__(kwargs.get('artifact_types'), - kwargs.get('filter_string'), - kwargs.get('allowed_sources')) - self.artifact_types = kwargs.get('artifact_types') or [ - threatingestor.artifacts.URL, - threatingestor.artifacts.Domain, - threatingestor.artifacts.Hash, - threatingestor.artifacts.IPAddress, - ] - - - def handle_artifact(self, artifact): - """Operate on a single artifact.""" - status = artifact.format_message(self.status) - - # If artifact.reference_link is a tweet permalink, quote-tweet it. - quote_tweet = None - if TWEET_URL.match(artifact.reference_link): - quote_tweet = artifact.reference_link - - self._tweet(status, quote_tweet=quote_tweet) - - - def _tweet(self, status, quote_tweet=None): - """Send content to Twitter as a status update.""" - try: - return self.api.statuses.update(status=status, - attachment_url=quote_tweet, - tweet_mode='extended') - except twitter.api.TwitterHTTPError as e: - logger.warning(f"Twitter API Error: {e}") - return None From 0a5e362cf5709763bc3220c0daa81431c731d0cb Mon Sep 17 00:00:00 2001 From: azazelm3dj3d <56496067+azazelm3dj3d@users.noreply.github.com> Date: Tue, 3 Oct 2023 15:18:05 -0500 Subject: [PATCH 2/3] Add BugnSnag to docs + some error handling --- config.example.yml | 4 +++ docs/extras.rst | 13 ++++++++ threatingestor/__init__.py | 54 ++++++++++++++++++++++++++------ threatingestor/bugsnagmonitor.py | 20 ------------ 4 files changed, 61 insertions(+), 30 deletions(-) delete mode 100644 threatingestor/bugsnagmonitor.py diff --git a/config.example.yml b/config.example.yml index 0eb167b..19c563b 100644 --- a/config.example.yml +++ b/config.example.yml @@ -46,6 +46,10 @@ credentials: key: MY_API_KEY ssl: False +error_reporting: + - name: bugsnag + api_key: API_KEY + sources: # This section defines each of the input sources for ThreatIngestor. # Define as many as you want. ThreatIngestor maintains a "state" for each of diff --git a/docs/extras.rst b/docs/extras.rst index fc6f175..761708a 100644 --- a/docs/extras.rst +++ b/docs/extras.rst @@ -3,6 +3,19 @@ Extras There are a few extra tools included alongside ThreatIngestor, that didn't quite make sense as sources or operators. +BugSnag +------- + +BugSnag monitoring is a valuable tool to have when running in daemon mode. Adding it is extremely simple and the only requirement is to have an API key. + +Here's an example to include in your ``config.yml``: + +.. code-block:: yaml + + error_reporting: + - name: bugsnag + api_key: API_KEY + Quick Webapp ------------ diff --git a/threatingestor/__init__.py b/threatingestor/__init__.py index c8f8c22..d931efe 100644 --- a/threatingestor/__init__.py +++ b/threatingestor/__init__.py @@ -12,12 +12,29 @@ logger.info("Notifiers is not installed.") notifiers = None +try: + import bugsnag + bugsnag_imported = True +except ImportError: + logger.info("BugSnag is not installed.") + bugsnag_imported = False + import threatingestor.config import threatingestor.state import threatingestor.exceptions import threatingestor.whitelist -from threatingestor.bugsnagmonitor import configure_bugsnag, send_notification +BUGSNAG_ACTIVE = False + +def bugsnag_notification(msg=None, metadata=None) -> None: + """ + Monitor your code with BugSnag + + You can include a additional information with the `metadata` paramater. + """ + + if bugsnag_imported: + bugsnag.notify(Exception(msg), metadata={"ThreatIngestor" : metadata}) class Ingestor: """ThreatIngestor main work logic. @@ -63,12 +80,17 @@ def __init__(self, config_file): except TypeError: logger.exception("Couldn't initialize statsd client; bad config?") sys.exit(1) - + # Configure BugSnag - for service in self.config.error_reporting(): - if service['name'] == "bugsnag": - configure_bugsnag(service['api_key']) - break + if bugsnag_imported: + for service in self.config.error_reporting(): + if service['name'] == "bugsnag": + if service['api_key']: + bugsnag.configure(api_key=service['api_key']) + logger.debug("BugSnag configured") + BUGSNAG_ACTIVE = True + + break # Load state DB. try: @@ -77,7 +99,10 @@ def __init__(self, config_file): except (OSError, IOError, threatingestor.exceptions.IngestorError): # Error loading state DB. logger.exception("Error reading state database") - send_notification("Error reading state database") + + if BUGSNAG_ACTIVE: + bugsnag_notification("Error reading state database") + sys.exit(1) # Instantiate plugins. @@ -95,7 +120,10 @@ def __init__(self, config_file): except (TypeError, ConnectionError, threatingestor.exceptions.PluginError): logger.exception("Error initializing plugins") - send_notification("Error initializing plugins") + + if BUGSNAG_ACTIVE: + bugsnag_notification("Error initializing plugins") + sys.exit(1) def _is_whitelisted(self, artifact) -> bool: @@ -132,7 +160,10 @@ def run_once(self): except Exception: self.statsd.incr(f'error.source.{source}') logger.exception(f"Unknown error in source '{source}'") - send_notification(f"Unknown error in source '{source}'") + + if BUGSNAG_ACTIVE: + bugsnag_notification(f"Unknown error in source '{source}'") + continue # Save the source state. @@ -155,7 +186,10 @@ def run_once(self): except Exception: self.statsd.incr(f'error.operator.{operator}') logger.exception(f"Unknown error in operator '{operator}'") - send_notification(f"Unknown error in operator '{operator}'") + + if BUGSNAG_ACTIVE: + bugsnag_notification(f"Unknown error in operator '{operator}'") + continue # Record stats and update the summary. diff --git a/threatingestor/bugsnagmonitor.py b/threatingestor/bugsnagmonitor.py deleted file mode 100644 index 216f40f..0000000 --- a/threatingestor/bugsnagmonitor.py +++ /dev/null @@ -1,20 +0,0 @@ -import bugsnag - -def configure_bugsnag(api_key=None) -> bool: - """ - Configure BugSnag API communication in `config.yml`. - """ - - if api_key: - bugsnag.configure(api_key=api_key) - return True - - return False - -def send_notification(msg=None, metadata=None) -> None: - """ - Monitor your code with BugSnag - - You can include a additional information with the `metadata` paramater. - """ - bugsnag.notify(Exception(msg), metadata={"ThreatIngestor" : metadata}) From ec98afdd9558f1b4da4a2986b66f67f2172e52b0 Mon Sep 17 00:00:00 2001 From: azazelm3dj3d <56496067+azazelm3dj3d@users.noreply.github.com> Date: Tue, 3 Oct 2023 15:24:57 -0500 Subject: [PATCH 3/3] Delete test_operators_twitter.py --- tests/test_operators_twitter.py | 65 --------------------------------- 1 file changed, 65 deletions(-) delete mode 100644 tests/test_operators_twitter.py diff --git a/tests/test_operators_twitter.py b/tests/test_operators_twitter.py deleted file mode 100644 index d6e727a..0000000 --- a/tests/test_operators_twitter.py +++ /dev/null @@ -1,65 +0,0 @@ -import unittest -from unittest.mock import patch, Mock - -import threatingestor.operators.twitter -import threatingestor.artifacts -import threatingestor.exceptions - - -class TestTwitter(unittest.TestCase): - - @patch('twitter.Twitter') - def setUp(self, Twitter): - self.twitter = threatingestor.operators.twitter.Plugin('a', 'b', 'c', 'd', 'status: {artifact}') - - def test_handle_artifact_interpolates_status_message(self): - self.twitter._tweet = Mock() - artifact = threatingestor.artifacts.URL('http://somedomain.com/test', 'test') - expected_content = 'status: http://somedomain.com/test' - self.twitter.handle_artifact(artifact) - self.twitter._tweet.assert_called_once_with(expected_content, quote_tweet=None) - - def test_handle_artifact_passes_quote_tweet_for_tweet_links(self): - self.twitter._tweet = Mock() - artifact = threatingestor.artifacts.URL('http://somedomain.com/test', 'test', - 'https://twitter.com/InQuest/status/00000000000') - expected_content = 'status: http://somedomain.com/test' - self.twitter.handle_artifact(artifact) - self.twitter._tweet.assert_called_once_with(expected_content, - quote_tweet='https://twitter.com/InQuest/status/00000000000') - - # make sure quote_tweet doesn't get set if the link isn't a tweet permalink - self.twitter._tweet.reset_mock() - artifact = threatingestor.artifacts.URL('http://somedomain.com/test', 'test', - 'https://twitter.com/help/') - expected_content = 'status: http://somedomain.com/test' - self.twitter.handle_artifact(artifact) - self.twitter._tweet.assert_called_once_with(expected_content, - quote_tweet=None) - - def test_artifact_types_are_set_if_passed_in_else_default(self): - artifact_types = [ - threatingestor.artifacts.URL, - threatingestor.artifacts.Domain, - threatingestor.artifacts.Hash, - threatingestor.artifacts.IPAddress, - ] - self.assertEqual(threatingestor.operators.twitter.Plugin('a', 'b', 'c', 'd', 'e', - artifact_types=[threatingestor.artifacts.URL]).artifact_types, - [threatingestor.artifacts.URL]) - self.assertEqual(threatingestor.operators.twitter.Plugin('a', 'b', 'c', 'd', 'e').artifact_types, - artifact_types) - - def test_init_sets_config_args(self): - operator = threatingestor.operators.twitter.Plugin('a', 'b', 'c', 'd', 'e', - filter_string='test', - allowed_sources=['test-one']) - self.assertEqual(operator.filter_string, 'test') - self.assertEqual(operator.allowed_sources, ['test-one']) - - def test_init_raises_if_bad_status(self): - with self.assertRaises(threatingestor.exceptions.IngestorError): - operator = threatingestor.operators.twitter.Plugin('a', 'b', 'c', 'd', - filter_string='test', - allowed_sources=['test-one'], - status=[])