Merge pull request #13 from obsidianforensics/refactor

Refactoring code to multiple files and package into 'pyhindsight' for…

MANIFEST.in

@@ -0,0 +1,5 @@
+recursive-include pyhindsight/plugins *.py
+recursive-include pyhindsight/static *
+recursive-include pyhindsight/templates *.tpl
+include hindsight.py
+include hindsight_gui.py

hindsight_gui.py

@@ -1,25 +1,59 @@
-import hindsight
-import bottle
 import json
 import os
 import sys
-import webbrowser
+import logging
+import bottle
+import importlib
+import pyhindsight
+import pyhindsight.plugins
+from pyhindsight.analysis import AnalysisSession
+from pyhindsight.utils import banner, MyEncoder
 
-# This will be the main hindsight.AnalysisSession object that all the work will be done on
+# This will be the main pyhindsight.AnalysisSession object that all the work will be done on
 analysis_session = None
+STATIC_PATH = 'static'
 
 
 def get_plugins_info():
 
     plugin_descriptions = []
+    completed_plugins = []
+
+    # First run built-in plugins that ship with Hindsight
+    logging.info(" Built-in Plugins:")
+    for plugin in pyhindsight.plugins.__all__:
+        # Check to see if we've already run this plugin (likely from a different path)
+        if plugin in completed_plugins:
+            continue
+
+        description = {'file_name': plugin, 'friendly_name': None, 'version': None, 'error': None,
+                       'error_msg': None, 'parent_path': None}
+
+        try:
+            module = importlib.import_module("pyhindsight.plugins.{}".format(plugin))
+            description['friendly_name'] = module.friendlyName
+            description['version'] = module.version
+            try:
+                module.plugin()
+            except ImportError, e:
+                description['error'] = 'import'
+                description['error_msg'] = e
+                continue
+
+        except Exception, e:
+            description['error'] = 'other'
+            description['error_msg'] = e
+            continue
+
+        finally:
+            plugin_descriptions.append(description)
+            completed_plugins.append(plugin)
 
     # Useful when Hindsight is run from a different directory than where the file is located
     real_path = os.path.dirname(os.path.realpath(sys.argv[0]))
     if real_path not in sys.path:
         sys.path.insert(0, real_path)
 
-    completed_plugins = []
-
     # Loop through all paths, to pick up all potential locations for plugins
     for potential_path in sys.path:
         # If a subdirectory exists called 'plugins' at the current path, continue on
@@ -33,8 +67,10 @@ def get_plugins_info():
                 plugin_listing = os.listdir(potential_plugin_path)
 
                 for plugin in plugin_listing:
-                    if plugin[-3:] == ".py":
-                        description = {'file_name': plugin, 'friendly_name': None, 'version': None, 'error': None, 'error_msg': None}
+                    if plugin[-3:] == ".py" and plugin[0] != '_':
+
+                        description = {'file_name': plugin, 'friendly_name': None, 'version': None, 'error': None,
+                                       'error_msg': None, 'parent_path': potential_plugin_path}
                         plugin = plugin.replace(".py", "")
 
                         # Check to see if we've already run this plugin (likely from a different path)
@@ -80,10 +116,10 @@ def images(filename):
 def main_screen():
 
     global analysis_session
-    analysis_session = hindsight.AnalysisSession()
+    analysis_session = AnalysisSession()
     bottle_args = analysis_session.__dict__
-    plugins_info = get_plugins_info()
-    bottle_args['plugins_info'] = plugins_info
+    analysis_session.plugin_descriptions = get_plugins_info()
+    bottle_args['plugins_info'] = analysis_session.plugin_descriptions
     return bottle.template(os.path.join('templates', 'run.tpl'), bottle_args)
 
 
@@ -98,6 +134,15 @@ def do_run():
     analysis_session.timezone = bottle.request.forms.get('timezone')
     analysis_session.log_path = bottle.request.forms.get('log_path')
 
+    # Set up logging
+    logging.basicConfig(filename=analysis_session.log_path, level=logging.DEBUG,
+                        format='%(asctime)s.%(msecs).03d | %(message)s',
+                        datefmt='%Y-%m-%d %H:%M:%S')
+    # Hindsight version info
+    logging.info(
+        '\n' + '#' * 80 + '\n###    Hindsight v{} (https://github.com/obsidianforensics/hindsight)    ###\n'
+        .format(pyhindsight.__version__) + '#' * 80)
+
     if 'windows' in ui_selected_decrypts:
         analysis_session.available_decrypts['windows'] = 1
     else:
@@ -132,7 +177,7 @@ def generate_sqlite():
         # temp file deletion failed
         pass
 
-    analysis_session.generate_sqlite(output_object=temp_output)
+    analysis_session.generate_sqlite(temp_output)
     import StringIO
     str_io = StringIO.StringIO()
     with open(temp_output, 'rb') as f:
@@ -166,7 +211,7 @@ def generate_xlsx():
 def generate_json():
     import StringIO
     strIO = StringIO.StringIO()
-    strIO.write(json.dumps(analysis_session, cls=hindsight.MyEncoder, indent=4))
+    strIO.write(json.dumps(analysis_session, cls=MyEncoder, indent=4))
     strIO.seek(0)
     bottle.response.headers['Content-Type'] = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet; charset=UTF-8'
     bottle.response.headers['Content-Disposition'] = 'attachment; filename={}.json'.format(analysis_session.output_name)
@@ -175,10 +220,12 @@ def generate_json():
 
 def main():
 
-    print hindsight.banner
-
+    print banner
     global STATIC_PATH
-    STATIC_PATH = 'static'
+
+    # Get the hindsight module's path on disk to add to sys.path, so we can find templates and static files
+    module_path = os.path.dirname(pyhindsight.__file__)
+    sys.path.insert(0, module_path)
 
     # Loop through all paths in system path, to pick up all potential locations for templates and static files.
     # Paths can get weird when the program is run from a different directory, or when the packaged exe is unpacked.

pyhindsight/browsers/brave.py

@@ -0,0 +1,143 @@
+import logging
+import os
+import json
+import re
+from pyhindsight.browsers.chrome import Chrome
+from pyhindsight.utils import to_datetime
+
+
+class Brave(Chrome):
+    def __init__(self, profile_path, timezone=None):
+        Chrome.__init__(self, profile_path, browser_name=None, version=None, timezone=timezone, parsed_artifacts=None,
+                        installed_extensions=None, artifacts_counts=None)
+        self.browser_name = "Brave"
+
+    def get_history(self, path, history_file, version, row_type):
+        # Set up empty return array
+        results = []
+
+        logging.info("History items from {}:".format(history_file))
+        try:
+
+            with open(os.path.join(path, history_file), 'rb') as history_input:
+                history_raw = history_input.read()
+                history_json = json.loads(history_raw)
+
+                for version_dict in history_json['about']['brave']['versionInformation']:
+                    if version_dict['name'] == u'Brave':
+                        self.display_version = version_dict['version']
+
+                for s, site in enumerate(history_json['sites']):
+                    if history_json['sites'][s].get('location'):
+                        last_accessed = history_json['sites'][s]['lastAccessedTime'] if history_json['sites'][s].get('lastAccessedTime') else history_json['sites'][s]['lastAccessed']
+
+                        new_row = Brave.URLItem(s, history_json['sites'][s]['location'],
+                                                history_json['sites'][s].get('title', "<No Title>"),
+                                                last_accessed, last_accessed,
+                                                None, None, None, None, None, None, None, None, None, )
+
+                        # Set the row type as determined earlier
+                        new_row.row_type = row_type
+
+                        # Set the row type as determined earlier
+                        new_row.timestamp = to_datetime(new_row.last_visit_time, self.timezone)
+
+                        # Add the new row to the results array
+                        results.append(new_row)
+
+            self.artifacts_counts[history_file] = len(results)
+            logging.info(" - Parsed {} items".format(len(results)))
+            self.parsed_artifacts.extend(results)
+
+        except:
+            logging.error(" - Error opening '{}'".format(os.path.join(path, history_file)))
+            self.artifacts_counts[history_file] = 'Failed'
+            return
+
+    def process(self):
+        supported_databases = ['History', 'Archived History', 'Web Data', 'Cookies', 'Login Data', 'Extension Cookies']
+        supported_subdirs = ['Local Storage', 'Extensions', 'Cache']
+        supported_jsons = ['Bookmarks']  # , 'Preferences']
+        supported_items = supported_databases + supported_subdirs + supported_jsons
+        logging.debug("Supported items: " + str(supported_items))
+        input_listing = os.listdir(self.profile_path)
+
+        logging.info("Found the following supported files or directories:")
+        for input_file in input_listing:
+            if input_file in supported_items:
+                logging.info(" - %s" % input_file)
+
+        # Process History files
+        custom_type_re = re.compile(r'__([A-z0-9\._]*)$')
+        for input_file in input_listing:
+            if re.search(r'session-store-', input_file):
+                row_type = u'url'
+                custom_type_m = re.search(custom_type_re, input_file)
+                if custom_type_m:
+                    row_type = u'url ({})'.format(custom_type_m.group(1))
+                # self.get_history(args.input, input_file, self.version, row_type)
+                self.get_history(self.profile_path, input_file, self.version, row_type)
+                display_type = 'URL' if not custom_type_m else 'URL ({})'.format(custom_type_m.group(1))
+                self.artifacts_display[input_file] = "{} records".format(display_type)
+                print self.format_processing_output("{} records".format(display_type),
+                                                    self.artifacts_counts[input_file])
+
+            if input_file == 'Partitions':
+                partitions = os.listdir(os.path.join(self.profile_path, input_file))
+                for partition in partitions:
+                    partition_path = os.path.join(self.profile_path, input_file, partition)
+                    partition_listing = os.listdir(os.path.join(self.profile_path, input_file, partition))
+                    if 'Cookies' in partition_listing:
+                        self.get_cookies(partition_path, 'Cookies', [47])  # Parse cookies like a modern Chrome version (v47)
+                        print self.format_processing_output("Cookie records ({})".format(partition), self.artifacts_counts['Cookies'])
+
+                    if 'Local Storage' in partition_listing:
+                        self.get_local_storage(partition_path, 'Local Storage')
+                        print self.format_processing_output("Local Storage records ({})".format(partition), self.artifacts_counts['Local Storage'])
+
+        # Version information is moved to after parsing history, as we read the version from the same file rather than detecting via SQLite table attributes
+        print self.format_processing_output("Detected {} version".format(self.browser_name), self.display_version)
+        logging.info("Detected {} version {}".format(self.browser_name, self.display_version))
+
+        if 'Cache' in input_listing:
+            self.get_cache(self.profile_path, 'Cache', row_type=u'cache')
+            self.artifacts_display['Cache'] = "Cache records"
+            print self.format_processing_output(self.artifacts_display['Cache'],
+                                                self.artifacts_counts['Cache'])
+        if 'GPUCache' in input_listing:
+            self.get_cache(self.profile_path, 'GPUCache', row_type=u'cache (gpu)')
+            self.artifacts_display['GPUCache'] = "GPU Cache records"
+            print self.format_processing_output(self.artifacts_display['GPUCache'],
+                                                self.artifacts_counts['GPUCache'])
+
+        if 'Cookies' in input_listing:
+            self.get_cookies(self.profile_path, 'Cookies', [47])  # Parse cookies like a modern Chrome version (v47)
+            self.artifacts_display['Cookies'] = "Cookie records"
+            print self.format_processing_output("Cookie records", self.artifacts_counts['Cookies'])
+
+        if 'Local Storage' in input_listing:
+            self.get_local_storage(self.profile_path, 'Local Storage')
+            self.artifacts_display['Local Storage'] = "Local Storage records"
+            print self.format_processing_output("Local Storage records", self.artifacts_counts['Local Storage'])
+
+        if 'Web Data' in input_listing:
+            self.get_autofill(self.profile_path, 'Web Data', [47])  # Parse autofill like a modern Chrome version (v47)
+            self.artifacts_display['Autofill'] = "Autofill records"
+            print self.format_processing_output(self.artifacts_display['Autofill'],
+                                                self.artifacts_counts['Autofill'])
+
+        if 'Preferences' in input_listing:
+            self.get_preferences(self.profile_path, 'Preferences')
+            self.artifacts_display['Preferences'] = "Preference Items"
+            print self.format_processing_output("Preference Items", self.artifacts_counts['Preferences'])
+
+        if 'UserPrefs' in input_listing:
+            self.get_preferences(self.profile_path, 'UserPrefs')
+            self.artifacts_display['UserPrefs'] = "UserPrefs Items"
+            print self.format_processing_output("UserPrefs Items", self.artifacts_counts['UserPrefs'])
+
+        # Destroy the cached key so that json serialization doesn't
+        # have a cardiac arrest on the non-unicode binary data.
+        self.cached_key = None
+
+        self.parsed_artifacts.sort()