Merge pull request #26 from obsidianforensics/refactor-input-parsing

Refactor input parsing

hindsight.py

@@ -60,7 +60,7 @@ def error(self, message):
         description=description,
         epilog=epi)
 
-    parser.add_argument('-i', '--input', help='Path to the Chrome(ium) "Default" directory', required=True)
+    parser.add_argument('-i', '--input', help='Path to the Chrome(ium) profile directory (typically "Default")', required=True)
     parser.add_argument('-o', '--output', help='Name of the output file (without extension)')
     parser.add_argument('-b', '--browser_type', help='Type of input files', default='Chrome',
                         choices=['Chrome', 'Brave'])
@@ -132,15 +132,38 @@ def write_sqlite(analysis_session):
         else:
             print("\n Database file \"{}\" already exists. Please choose a different output location.\n".format(output_file))
 
+    def find_browser_profiles(base_path):
+        """Search a path for browser profiles (only Chromium-based at the moment)."""
+        found_profile_paths = []
+        base_dir_listing = os.listdir(base_path)
+
+        # The 'History' and 'Cookies' SQLite files are kind of the minimum required for most
+        # Chrome analysis. This approach (checking the file names) is naive but should work.
+        if {'History', 'Cookies'}.issubset(base_dir_listing):
+            found_profile_paths.append(base_path)
+
+        # Only search sub dirs if the current dir is not a Profile (Profiles are not nested).
+        else:
+            for item in base_dir_listing:
+                item_path = os.path.join(base_path, item)
+                if os.path.isdir(item_path):
+                    profile_found_in_subdir = find_browser_profiles(item_path)
+                    if profile_found_in_subdir:
+                        found_profile_paths.extend(profile_found_in_subdir)
+
+        return found_profile_paths
+
     print(banner)
 
     # Useful when Hindsight is run from a different directory than where the file is located
     real_path = os.path.dirname(os.path.realpath(sys.argv[0]))
 
     # Set up the AnalysisSession object, and transfer the relevant input arguments to it
     analysis_session = AnalysisSession()
+
+    # parse_arguments needs the analysis_session as an input to set things like available decrypts
     args = parse_arguments(analysis_session)
-    analysis_session.profile_path = args.input
+
     if args.output:
         analysis_session.output_name = args.output
 
@@ -170,10 +193,17 @@ def write_sqlite(analysis_session):
     print(format_meta_output("Start time", str(datetime.datetime.now())[:-3]))
 
     # Read the input directory
+    analysis_session.input_path = args.input
     print(format_meta_output("Input directory", args.input))
     log.info("Reading files from %s" % args.input)
     input_listing = os.listdir(args.input)
     log.debug("Input directory contents: " + str(input_listing))
+
+    # Search input directory for browser profiles to analyze
+    input_profiles = find_browser_profiles(args.input)
+    log.info(" - Found {} browser profile(s): {}".format(len(input_profiles), input_profiles))
+    analysis_session.profile_paths = input_profiles
+
     print(format_meta_output("Output name", "{}.{}".format(analysis_session.output_name, analysis_session.selected_output_format)))
 
     # Run the AnalysisSession

hindsight_gui.py

@@ -128,7 +128,7 @@ def do_run():
     # Get user selections from the UI
     ui_selected_decrypts = bottle.request.forms.getall('selected_decrypts')
     analysis_session.selected_plugins = bottle.request.forms.getall('selected_plugins')
-    analysis_session.profile_path = bottle.request.forms.get('profile_path')
+    analysis_session.input_path = bottle.request.forms.get('profile_path')  # TODO: refactor bottle name
     analysis_session.cache_path = bottle.request.forms.get('cache_path')
     analysis_session.browser_type = bottle.request.forms.get('browser_type')
     analysis_session.timezone = bottle.request.forms.get('timezone')

pyhindsight/__init__.py

@@ -1,3 +1,3 @@
 __author__ = "Ryan Benson"
-__version__ = "2.2.1"
+__version__ = "2.3.0"
 __email__ = "[email protected]"

pyhindsight/browsers/webbrowser.py

@@ -8,7 +8,7 @@
 
 class WebBrowser(object):
     def __init__(self, profile_path, browser_name, cache_path=None, version=None, display_version=None, timezone=None, structure=None,
-                 parsed_artifacts=None, artifacts_counts=None, artifacts_display=None):
+                 parsed_artifacts=None, artifacts_counts=None, artifacts_display=None, preferences=None):
         self.profile_path = profile_path
         self.browser_name = browser_name
         self.cache_path = cache_path
@@ -19,6 +19,7 @@ def __init__(self, profile_path, browser_name, cache_path=None, version=None, di
         self.parsed_artifacts = parsed_artifacts
         self.artifacts_counts = artifacts_counts
         self.artifacts_display = artifacts_display
+        self.preferences = preferences
         # self.logger = logger
 
         if self.version is None:
@@ -33,6 +34,9 @@ def __init__(self, profile_path, browser_name, cache_path=None, version=None, di
         if self.artifacts_display is None:
             self.artifacts_display = {}
 
+        if self.preferences is None:
+            self.preferences = []
+
     @staticmethod
     def format_processing_output(name, items):
         width = 80
@@ -43,6 +47,12 @@ def format_processing_output(name, items):
                     right_width=(width - int(left_side)-2))
         return pretty_name
 
+    @staticmethod
+    def format_profile_path(profile_path):
+        if len(profile_path) > 68:
+            profile_path = "...{}".format(profile_path[-65:])
+        return "\n    Profile: {}".format(profile_path)
+
     def build_structure(self, path, database):
 
         if database not in self.structure.keys():
@@ -88,9 +98,10 @@ def dict_factory(cursor, row):
         return d
 
     class HistoryItem(object):
-        def __init__(self, item_type, timestamp, url=None, name=None, value=None, interpretation=None):
+        def __init__(self, item_type, timestamp, profile, url=None, name=None, value=None, interpretation=None):
             self.row_type = item_type
             self.timestamp = timestamp
+            self.profile = profile
             self.url = url
             self.name = name
             self.value = value
@@ -103,10 +114,11 @@ def __iter__(self):
             return iter(self.__dict__)
 
     class URLItem(HistoryItem):
-        def __init__(self, url_id, url, title, visit_time, last_visit_time, visit_count, typed_count, from_visit,
+        def __init__(self, profile, url_id, url, title, visit_time, last_visit_time, visit_count, typed_count, from_visit,
                      transition, hidden, favicon_id, indexed=None, visit_duration=None, visit_source=None,
                      transition_friendly=None):
-            super(WebBrowser.URLItem, self).__init__('url', timestamp=visit_time, url=url, name=title)
+            super(WebBrowser.URLItem, self).__init__('url', timestamp=visit_time, profile=profile, url=url, name=title)
+            self.profile = profile
             self.url_id = url_id
             self.url = url
             self.title = title
@@ -124,11 +136,12 @@ def __init__(self, url_id, url, title, visit_time, last_visit_time, visit_count,
             self.transition_friendly = transition_friendly
 
     class DownloadItem(HistoryItem):
-        def __init__(self, download_id, url, received_bytes, total_bytes, state, full_path=None, start_time=None,
+        def __init__(self, profile, download_id, url, received_bytes, total_bytes, state, full_path=None, start_time=None,
                      end_time=None, target_path=None, current_path=None, opened=None, danger_type=None,
                      interrupt_reason=None, etag=None, last_modified=None, chain_index=None, interrupt_reason_friendly=None,
                      danger_type_friendly=None, state_friendly=None, status_friendly=None):
-            super(WebBrowser.DownloadItem, self).__init__(u'download', timestamp=start_time, url=url)
+            super(WebBrowser.DownloadItem, self).__init__(u'download', timestamp=start_time, profile=profile, url=url)
+            self.profile = profile
             self.download_id = download_id
             self.url = url
             self.received_bytes = received_bytes
@@ -151,9 +164,10 @@ def __init__(self, download_id, url, received_bytes, total_bytes, state, full_pa
             self.status_friendly = status_friendly
 
     class CookieItem(HistoryItem):
-        def __init__(self, host_key, path, name, value, creation_utc, last_access_utc, expires_utc, secure, http_only,
+        def __init__(self, profile, host_key, path, name, value, creation_utc, last_access_utc, expires_utc, secure, http_only,
                      persistent=None, has_expires=None, priority=None):
-            super(WebBrowser.CookieItem, self).__init__('cookie', timestamp=creation_utc, url=host_key, name=name, value=value)
+            super(WebBrowser.CookieItem, self).__init__('cookie', timestamp=creation_utc, profile=profile, url=host_key, name=name, value=value)
+            self.profile = profile
             self.host_key = host_key
             self.path = path
             self.name = name
@@ -168,58 +182,65 @@ def __init__(self, host_key, path, name, value, creation_utc, last_access_utc, e
             self.priority = priority
 
     class AutofillItem(HistoryItem):
-        def __init__(self, date_created, name, value, count):
-            super(WebBrowser.AutofillItem, self).__init__(u'autofill', timestamp=date_created, name=name, value=value)
+        def __init__(self, profile, date_created, name, value, count):
+            super(WebBrowser.AutofillItem, self).__init__(u'autofill', timestamp=date_created, profile=profile, name=name, value=value)
+            self.profile = profile
             self.date_created = date_created
             self.name = name
             self.value = value
             self.count = count
 
     class BookmarkItem(HistoryItem):
-        def __init__(self, date_added, name, url, parent_folder, sync_transaction_version=None):
-            super(WebBrowser.BookmarkItem, self).__init__(u'bookmark', timestamp=date_added, name=name, value=parent_folder)
+        def __init__(self, profile, date_added, name, url, parent_folder, sync_transaction_version=None):
+            super(WebBrowser.BookmarkItem, self).__init__(u'bookmark', timestamp=date_added, profile=profile, name=name, value=parent_folder)
+            self.profile = profile
             self.date_added = date_added
             self.name = name
             self.url = url
             self.parent_folder = parent_folder
             self.sync_transaction_version = sync_transaction_version
 
     class BookmarkFolderItem(HistoryItem):
-        def __init__(self, date_added, date_modified, name, parent_folder, sync_transaction_version=None):
-            super(WebBrowser.BookmarkFolderItem, self).__init__(u'bookmark folder', timestamp=date_added, name=name, value=parent_folder)
+        def __init__(self, profile, date_added, date_modified, name, parent_folder, sync_transaction_version=None):
+            super(WebBrowser.BookmarkFolderItem, self).__init__(u'bookmark folder', timestamp=date_added, profile=profile, name=name, value=parent_folder)
+            self.profile = profile
             self.date_added = date_added
             self.date_modified = date_modified
             self.name = name
             self.parent_folder = parent_folder
             self.sync_transaction_version = sync_transaction_version
 
     class LocalStorageItem(HistoryItem):
-        def __init__(self, url, date_created, key, value):
-            super(WebBrowser.LocalStorageItem, self).__init__(u'local storage', timestamp=date_created, name=key, value=value)
+        def __init__(self, profile, url, date_created, key, value):
+            super(WebBrowser.LocalStorageItem, self).__init__(u'local storage', timestamp=date_created, profile=profile, name=key, value=value)
+            self.profile = profile
             self.url = url
             self.date_created = date_created
             self.key = key
             self.value = value
 
     class BrowserExtension(object):
-        def __init__(self, app_id, name, description, version):
+        def __init__(self, profile, app_id, name, description, version):
+            self.profile = profile
             self.app_id = app_id
             self.name = name
             self.description = description
             self.version = version
 
     class LoginItem(HistoryItem):
-        def __init__(self, date_created, url, name, value, count):
-            super(WebBrowser.LoginItem, self).__init__(u'login', timestamp=date_created, url=url, name=name, value=value)
+        def __init__(self, profile, date_created, url, name, value, count):
+            super(WebBrowser.LoginItem, self).__init__(u'login', timestamp=date_created, profile=profile, url=url, name=name, value=value)
+            self.profile = profile
             self.date_created = date_created
             self.url = url
             self.name = name
             self.value = value
             self.count = count
 
     class PreferenceItem(HistoryItem):
-        def __init__(self, url, timestamp, key, value, interpretation):
-            super(WebBrowser.PreferenceItem, self).__init__(u'preference', timestamp=timestamp, name=key, value=value)
+        def __init__(self, profile, url, timestamp, key, value, interpretation):
+            super(WebBrowser.PreferenceItem, self).__init__(u'preference', timestamp=timestamp, profile=profile, name=key, value=value)
+            self.profile = profile
             self.url = url
             self.timestamp = timestamp
             self.key = key

pyhindsight/templates/header.tpl

@@ -30,7 +30,7 @@
                 <td align="right">
                     <span class="social-logos logo-bars">
                         <a href="https://github.com/obsidianforensics/hindsight" target="_blank"><img class="social-logo" src="static/github.png" title="View the code on GitHub"></a> |
-                        <a href="http://slack.hindsig.ht/" target="_blank"><img class="social-logo" src="static/slack.png" title="Join the Hindsight Slack Channel"></a> |
+                        <a href="https://join-open-source-dfir-slack.herokuapp.com/" target="_blank"><img class="social-logo" src="static/slack.png" title="Join the Hindsight Slack Channel"></a> |
                         <a href="https://twitter.com/_RyanBenson" target="_blank"><img class="social-logo" src="static/twitter.png" title="Say Hi!"></a>
                     </span>
                 </td>

pyhindsight/templates/options_results.tpl

@@ -2,9 +2,16 @@
     <div class="header-box">Summary</div>
     <div class="selection-box">
          <table width=100%>
-            <tr><td align="right">Input Path:</td><td>{{profile_path}}</td></tr>
-            <tr><td align="right">Input Type:</td><td>{{browser_type}}</td></tr>
-
+            <tr><td align="right" colspan=2>Input Path:</td><td>{{input_path}}</td></tr>
+            <tr><td align="right" colspan=2>Input Type:</td><td>{{browser_type}}</td></tr>
+            <tr><td align="right" colspan=2>Profile Paths:</td><td></td></tr>
+            <tr><td></td><td colspan=2>
+                <ul>
+            % for path in profile_paths:
+                    <li>{{path}}</li>
+            % end
+                </ul>
+            </td></tr>
           </table>
     </div>
 </div>

pyhindsight/templates/parsed_artifacts.tpl

@@ -8,13 +8,13 @@
                 <td align="right">{{display_version}}</td>
                 <td width=10%></td>
             </tr>
-         % display_items = artifacts_counts.keys()
+         % display_items = artifacts_display.keys()
          % display_order = ['Archived History', 'History', 'History_downloads', 'Cache', 'Application Cache', 'Media Cache', 'GPUCache', 'Cookies', 'Local Storage', 'Bookmarks', 'Autofill', 'Login Data', 'Preferences', 'Extensions', 'Extension Cookies' ]
          % while len(display_order) > 0:
          %   if display_order[0] in display_items:
             <tr class="results-row">
                 <td align="right">{{artifacts_display[display_order[0]]}}:</td>
-                <td align="right">{{artifacts_counts[display_order[0]]}}</td>
+                <td align="right">{{artifacts_counts.get(display_order[0], 0)}}</td>
                 <td width=10%></td>
 
             </tr>

requirements.txt

@@ -1,3 +1,4 @@
+leveldb>=0.1
 keyring>=9.0
 matplotlib>=1.5.1
 pytz>=2016.4