diff --git a/config.example.yml b/config.example.yml index 0eb167b..19c563b 100644 --- a/config.example.yml +++ b/config.example.yml @@ -46,6 +46,10 @@ credentials: key: MY_API_KEY ssl: False +error_reporting: + - name: bugsnag + api_key: API_KEY + sources: # This section defines each of the input sources for ThreatIngestor. # Define as many as you want. ThreatIngestor maintains a "state" for each of diff --git a/docs/extras.rst b/docs/extras.rst index fc6f175..761708a 100644 --- a/docs/extras.rst +++ b/docs/extras.rst @@ -3,6 +3,19 @@ Extras There are a few extra tools included alongside ThreatIngestor, that didn't quite make sense as sources or operators. +BugSnag +------- + +BugSnag monitoring is a valuable tool to have when running in daemon mode. Adding it is extremely simple and the only requirement is to have an API key. + +Here's an example to include in your ``config.yml``: + +.. code-block:: yaml + + error_reporting: + - name: bugsnag + api_key: API_KEY + Quick Webapp ------------ diff --git a/tests/test_operators_twitter.py b/tests/test_operators_twitter.py deleted file mode 100644 index d6e727a..0000000 --- a/tests/test_operators_twitter.py +++ /dev/null @@ -1,65 +0,0 @@ -import unittest -from unittest.mock import patch, Mock - -import threatingestor.operators.twitter -import threatingestor.artifacts -import threatingestor.exceptions - - -class TestTwitter(unittest.TestCase): - - @patch('twitter.Twitter') - def setUp(self, Twitter): - self.twitter = threatingestor.operators.twitter.Plugin('a', 'b', 'c', 'd', 'status: {artifact}') - - def test_handle_artifact_interpolates_status_message(self): - self.twitter._tweet = Mock() - artifact = threatingestor.artifacts.URL('http://somedomain.com/test', 'test') - expected_content = 'status: http://somedomain.com/test' - self.twitter.handle_artifact(artifact) - self.twitter._tweet.assert_called_once_with(expected_content, quote_tweet=None) - - def test_handle_artifact_passes_quote_tweet_for_tweet_links(self): - self.twitter._tweet = Mock() - artifact = threatingestor.artifacts.URL('http://somedomain.com/test', 'test', - 'https://twitter.com/InQuest/status/00000000000') - expected_content = 'status: http://somedomain.com/test' - self.twitter.handle_artifact(artifact) - self.twitter._tweet.assert_called_once_with(expected_content, - quote_tweet='https://twitter.com/InQuest/status/00000000000') - - # make sure quote_tweet doesn't get set if the link isn't a tweet permalink - self.twitter._tweet.reset_mock() - artifact = threatingestor.artifacts.URL('http://somedomain.com/test', 'test', - 'https://twitter.com/help/') - expected_content = 'status: http://somedomain.com/test' - self.twitter.handle_artifact(artifact) - self.twitter._tweet.assert_called_once_with(expected_content, - quote_tweet=None) - - def test_artifact_types_are_set_if_passed_in_else_default(self): - artifact_types = [ - threatingestor.artifacts.URL, - threatingestor.artifacts.Domain, - threatingestor.artifacts.Hash, - threatingestor.artifacts.IPAddress, - ] - self.assertEqual(threatingestor.operators.twitter.Plugin('a', 'b', 'c', 'd', 'e', - artifact_types=[threatingestor.artifacts.URL]).artifact_types, - [threatingestor.artifacts.URL]) - self.assertEqual(threatingestor.operators.twitter.Plugin('a', 'b', 'c', 'd', 'e').artifact_types, - artifact_types) - - def test_init_sets_config_args(self): - operator = threatingestor.operators.twitter.Plugin('a', 'b', 'c', 'd', 'e', - filter_string='test', - allowed_sources=['test-one']) - self.assertEqual(operator.filter_string, 'test') - self.assertEqual(operator.allowed_sources, ['test-one']) - - def test_init_raises_if_bad_status(self): - with self.assertRaises(threatingestor.exceptions.IngestorError): - operator = threatingestor.operators.twitter.Plugin('a', 'b', 'c', 'd', - filter_string='test', - allowed_sources=['test-one'], - status=[]) diff --git a/threatingestor/__init__.py b/threatingestor/__init__.py index 67a1e6c..d931efe 100644 --- a/threatingestor/__init__.py +++ b/threatingestor/__init__.py @@ -12,11 +12,30 @@ logger.info("Notifiers is not installed.") notifiers = None +try: + import bugsnag + bugsnag_imported = True +except ImportError: + logger.info("BugSnag is not installed.") + bugsnag_imported = False + import threatingestor.config import threatingestor.state import threatingestor.exceptions import threatingestor.whitelist +BUGSNAG_ACTIVE = False + +def bugsnag_notification(msg=None, metadata=None) -> None: + """ + Monitor your code with BugSnag + + You can include a additional information with the `metadata` paramater. + """ + + if bugsnag_imported: + bugsnag.notify(Exception(msg), metadata={"ThreatIngestor" : metadata}) + class Ingestor: """ThreatIngestor main work logic. @@ -35,6 +54,7 @@ def __init__(self, config_file): # Configure logging with optional notifiers. logger.configure(**self.config.logging()) + try: logger.level("NOTIFY", no=35, color="", icon="\U0001F514") except TypeError: @@ -61,6 +81,17 @@ def __init__(self, config_file): logger.exception("Couldn't initialize statsd client; bad config?") sys.exit(1) + # Configure BugSnag + if bugsnag_imported: + for service in self.config.error_reporting(): + if service['name'] == "bugsnag": + if service['api_key']: + bugsnag.configure(api_key=service['api_key']) + logger.debug("BugSnag configured") + BUGSNAG_ACTIVE = True + + break + # Load state DB. try: logger.debug(f"Opening state database '{self.config.state_path()}'") @@ -68,6 +99,10 @@ def __init__(self, config_file): except (OSError, IOError, threatingestor.exceptions.IngestorError): # Error loading state DB. logger.exception("Error reading state database") + + if BUGSNAG_ACTIVE: + bugsnag_notification("Error reading state database") + sys.exit(1) # Instantiate plugins. @@ -84,8 +119,11 @@ def __init__(self, config_file): self.whitelist = threatingestor.whitelist.Whitelist(self.config.whitelists()) except (TypeError, ConnectionError, threatingestor.exceptions.PluginError): - logger.warning("Twitter config format has recently changed. See https://github.com/InQuest/ThreatIngestor/releases/tag/v1.0.0b5") logger.exception("Error initializing plugins") + + if BUGSNAG_ACTIVE: + bugsnag_notification("Error initializing plugins") + sys.exit(1) def _is_whitelisted(self, artifact) -> bool: @@ -122,6 +160,10 @@ def run_once(self): except Exception: self.statsd.incr(f'error.source.{source}') logger.exception(f"Unknown error in source '{source}'") + + if BUGSNAG_ACTIVE: + bugsnag_notification(f"Unknown error in source '{source}'") + continue # Save the source state. @@ -144,6 +186,10 @@ def run_once(self): except Exception: self.statsd.incr(f'error.operator.{operator}') logger.exception(f"Unknown error in operator '{operator}'") + + if BUGSNAG_ACTIVE: + bugsnag_notification(f"Unknown error in operator '{operator}'") + continue # Record stats and update the summary. diff --git a/threatingestor/config.py b/threatingestor/config.py index 5a96566..c8bc9d7 100644 --- a/threatingestor/config.py +++ b/threatingestor/config.py @@ -77,6 +77,11 @@ def logging(self): return self.config.get('logging', {}) + def error_reporting(self): + """Returns error_reporting config dictionary.""" + return self.config['error_reporting'] + + def credentials(self, credential_name): """Return a dictionary with the specified credentials.""" for credential in self.config['credentials']: diff --git a/threatingestor/operators/twitter.py b/threatingestor/operators/twitter.py deleted file mode 100644 index 19a83c1..0000000 --- a/threatingestor/operators/twitter.py +++ /dev/null @@ -1,54 +0,0 @@ -from __future__ import absolute_import - -import re -import twitter -from loguru import logger - -from threatingestor.operators import Operator -import threatingestor.artifacts -import threatingestor.exceptions - -TWEET_URL = re.compile(r'https://twitter\.com/\w{1,15}/status/\d+') - -class Plugin(Operator): - """Operator for Twitter.""" - def __init__(self, api_key, api_secret_key, access_token, access_token_secret, status, **kwargs): - self.api = twitter.Twitter(auth=twitter.OAuth(access_token, access_token_secret, api_key, api_secret_key)) - self.status = status - - # Validate status, for better error handling. - if not isinstance(self.status, str): - raise threatingestor.exceptions.IngestorError(f"Invalid 'status' config: {self.status}") - - super(Plugin, self).__init__(kwargs.get('artifact_types'), - kwargs.get('filter_string'), - kwargs.get('allowed_sources')) - self.artifact_types = kwargs.get('artifact_types') or [ - threatingestor.artifacts.URL, - threatingestor.artifacts.Domain, - threatingestor.artifacts.Hash, - threatingestor.artifacts.IPAddress, - ] - - - def handle_artifact(self, artifact): - """Operate on a single artifact.""" - status = artifact.format_message(self.status) - - # If artifact.reference_link is a tweet permalink, quote-tweet it. - quote_tweet = None - if TWEET_URL.match(artifact.reference_link): - quote_tweet = artifact.reference_link - - self._tweet(status, quote_tweet=quote_tweet) - - - def _tweet(self, status, quote_tweet=None): - """Send content to Twitter as a status update.""" - try: - return self.api.statuses.update(status=status, - attachment_url=quote_tweet, - tweet_mode='extended') - except twitter.api.TwitterHTTPError as e: - logger.warning(f"Twitter API Error: {e}") - return None