diff --git a/.gitignore b/.gitignore index 72364f99..bd7300ba 100644 --- a/.gitignore +++ b/.gitignore @@ -85,5 +85,7 @@ ENV/ # Spyder project settings .spyderproject +# Pycharm +.idea/ # Rope project settings .ropeproject diff --git a/plugins/lighthouse/coverage.py b/plugins/lighthouse/coverage.py index f3ac1411..c4beadbb 100644 --- a/plugins/lighthouse/coverage.py +++ b/plugins/lighthouse/coverage.py @@ -8,13 +8,14 @@ from lighthouse.util import * from lighthouse.util.qt import compute_color_on_gradiant -from lighthouse.metadata import DatabaseMetadata +from lighthouse.metadata import DatabaseMetadata, Locker logger = logging.getLogger("Lighthouse.Coverage") -#------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ # Coverage Mapping -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ # # When raw runtime data (eg, coverage or trace data) is passed into the # director, it is stored internally in DatabaseCoverage objects. A @@ -33,9 +34,9 @@ # get updated or refreshed by the user. # -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ # Database Coverage -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ class DatabaseCoverage(object): """ @@ -177,9 +178,9 @@ def __init__(self, palette, name="", filepath=None, data=None): self._weak_self = weakref.proxy(self) - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Properties - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- @property def data(self): @@ -220,7 +221,7 @@ def suspicious(self): bad += 1 # compute a percentage of the 'bad nodes' - percent = (bad/float(total))*100 + percent = (bad / float(total)) * 100 logger.debug("SUSPICIOUS: %5.2f%% (%u/%u)" % (percent, bad, total)) # @@ -234,9 +235,9 @@ def suspicious(self): return percent > 2.0 - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Metadata Population - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def update_metadata(self, metadata, delta=None): """ @@ -277,7 +278,7 @@ def update_metadata(self, metadata, delta=None): # elif rebase_offset: - self._hitmap = { (address + rebase_offset): hits for address, hits in iteritems(self._hitmap) } + self._hitmap = {(address + rebase_offset): hits for address, hits in iteritems(self._hitmap)} self._imagebase = self._metadata.imagebase # @@ -302,7 +303,7 @@ def refresh(self): self._update_coverage_hash() # dump the unmappable coverage data - #self.dump_unmapped() + # self.dump_unmapped() def refresh_theme(self): """ @@ -369,9 +370,9 @@ def _finalize_instruction_percent(self): # save the computed percentage of database instructions executed (0 to 1.0) self.instruction_percent = float(executed) / total - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Data Operations - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def add_data(self, data, update=True): """ @@ -464,9 +465,9 @@ def _update_coverage_hash(self): else: self.coverage_hash = 0 - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Coverage Mapping - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def _map_coverage(self): """ @@ -648,33 +649,34 @@ def unmap_all(self): self._unmapped_data = set(self._hitmap.keys()) self._unmapped_data.add(BADADDR) - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Debug - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def dump_unmapped(self): """ Dump the unmapped coverage data. """ lmsg("Unmapped coverage data for %s:" % self.name) - if len(self._unmapped_data) == 1: # 1 is going to be BADADDR + if len(self._unmapped_data) == 1: # 1 is going to be BADADDR lmsg(" * (there is no unmapped data!)") return for address in self._unmapped_data: lmsg(" * 0x%X" % address) -#------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ # Function Coverage -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ class FunctionCoverage(object): """ Function level coverage mapping. """ - def __init__(self, function_address, database=None): - self.database = database + def __init__(self, function_address, metadata_database=None): + self.database = metadata_database self.address = function_address # addresses of nodes executed @@ -687,9 +689,66 @@ def __init__(self, function_address, database=None): # baked colors self.coverage_color = 0 - #-------------------------------------------------------------------------- + self._accumulated_instruction_executed = None + self._accumulated_edge_executed = None + self.__accumulated_edge_locker = Locker(False) + self.__accumulated_instruction_locker = Locker(False) + + # -------------------------------------------------------------------------- # Properties - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- + + def __coverage_accumulated_val(self, locker, internal_member_name, external_property_name, initial_value): + if self.__getattribute__(internal_member_name) is not None: + return self.__getattribute__(internal_member_name) + + if self.database is None: + if self.address == BADADDR: + logger.debug("coverage_variable_count") + else: + raise ValueError("can't resolve addr_to_function_coverage") + return 0 + + # There is a loop, return 0 because the real value of the count will be calculated in the locked part. + if locker.locked(): + logger.info( + "calculating accumulated value for {function_name} reached loop".format(function_name=hex(self.address))) + return 0 + + with locker as _: + accumulated_generic_count = initial_value + function_metadata = self.database._metadata.functions[self.address] + for func_addr in function_metadata.called_addresses: + if func_addr not in self.database.functions: + logger.warning("In function addr:{func_addr} called {external_func_addr} could not find code ref " + "object for coverage accumulated calculation - can it be external function?".format( + func_addr=hex(self.address), external_func_addr=hex(func_addr))) + continue + + metadata_called_func = self.database.functions[func_addr] + accumulated_generic_count += metadata_called_func.__getattribute__(external_property_name) + + + logger.debug("setting value {} in {}".format(accumulated_generic_count, internal_member_name)) + self.__setattr__(internal_member_name, accumulated_generic_count) + + return self.__getattribute__(internal_member_name) + + @property + def accumulated_edge_executed(self): + logger.info("called edge executed node_executed {}".format(self.nodes_executed)) + return self.__coverage_accumulated_val(self.__accumulated_edge_locker, + "_accumulated_edge_executed", + "accumulated_edge_executed", + self.nodes_executed) + + @property + def accumulated_instruction_executed(self): + logger.info("called instructions executed {}".format(self.instructions_executed)) + return self.__coverage_accumulated_val(self.__accumulated_edge_locker, + "_accumulated_instruction_executed", + "accumulated_instruction_executed", + self.instructions_executed) @property def hits(self): @@ -719,9 +778,9 @@ def instructions(self): """ return set([ea for node in itervalues(self.nodes) for ea in node.executed_instructions.keys()]) - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Controls - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def mark_node(self, node_coverage): """ @@ -754,10 +813,18 @@ def finalize(self): self.database.palette.table_coverage_bad, self.database.palette.table_coverage_good ) + logger.info("called finalize inst {} nodes {}".format(self.instructions_executed, len(self.nodes))) + # clean cache + with self.__accumulated_edge_locker as _: + self._accumulated_edge_executed = None + + with self.__accumulated_instruction_locker as _: + self._accumulated_instruction_executed = None + -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ # Node Coverage -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ class NodeCoverage(object): """ @@ -770,9 +837,9 @@ def __init__(self, node_address, database=None): self.executed_instructions = {} self.instructions_executed = 0 - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Properties - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- @property def hits(self): @@ -781,9 +848,9 @@ def hits(self): """ return sum(itervalues(self.executed_instructions)) - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Controls - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def finalize(self): """ diff --git a/plugins/lighthouse/metadata.py b/plugins/lighthouse/metadata.py index 23f974d6..10bb8776 100644 --- a/plugins/lighthouse/metadata.py +++ b/plugins/lighthouse/metadata.py @@ -16,9 +16,10 @@ logger = logging.getLogger("Lighthouse.Metadata") -#------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ # Metadata -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ # # To aid in performance, Lighthouse lifts and indexes an in-memory limited # representation of the disassembler's open database. This is commonly @@ -56,9 +57,9 @@ # reasonably low cost refresh. # -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ # Database Metadata -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ class DatabaseMetadata(object): """ @@ -103,23 +104,23 @@ def __init__(self, lctx=None): self._go_synchronous = False # a scheduled callback to watch for specific database changes - self._scheduled_interval = 2000 # ms + self._scheduled_interval = 2000 # ms self._scheduled_timer = QtCore.QTimer() self._scheduled_timer.setInterval(self._scheduled_interval) self._scheduled_timer.setSingleShot(True) self._scheduled_timer.timeout.connect(self._scheduled_worker) - #---------------------------------------------------------------------- + # ---------------------------------------------------------------------- # Callbacks - #---------------------------------------------------------------------- + # ---------------------------------------------------------------------- self._metadata_modified_callbacks = [] self._function_renamed_callbacks = [] self._rebased_callbacks = [] - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Subsystem Lifetime - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def start(self): """ @@ -148,16 +149,16 @@ def terminate(self): del self._rebased_callbacks self._clear_cache() - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Providers - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def get_instructions_slice(self, start_address, end_address): """ Get the instructions addresses that fall within a given range. """ index_start = bisect.bisect_left(self.instructions, start_address) - index_end = bisect.bisect_left(self.instructions, end_address) + index_end = bisect.bisect_left(self.instructions, end_address) return self.instructions[index_start:index_end] def get_instruction_size(self, address): @@ -216,7 +217,7 @@ def get_node(self, address): # if not (node_metadata and address in node_metadata.instructions): - node_metadata = self.nodes.get(self._node_addresses[index-1], None) + node_metadata = self.nodes.get(self._node_addresses[index - 1], None) # double fault, let's just dip... if not (node_metadata and address in node_metadata.instructions): @@ -300,7 +301,7 @@ def get_closest_function(self, address): # select the two candidate addresses before = self._function_addresses[index - 1] - after = self._function_addresses[index] + after = self._function_addresses[index] # return the function closest to the given address if after - address < address - before: @@ -314,9 +315,9 @@ def is_big(self): """ return len(self.functions) > 50000 - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Refresh - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def refresh(self, progress_callback=None): """ @@ -430,9 +431,9 @@ def _refresh_lookup(self): - get_function(ea) """ - self._last_node = lambda: None # XXX blank node hack, see other ref to _last_node + self._last_node = lambda: None # XXX blank node hack, see other ref to _last_node self._last_node.instructions = [] - self._name2func = { f.name: f.address for f in itervalues(self.functions) } + self._name2func = {f.name: f.address for f in itervalues(self.functions)} self._node_addresses = sorted(self.nodes.keys()) self._function_addresses = sorted(self.functions.keys()) for function_metadata in itervalues(self.functions): @@ -447,9 +448,9 @@ def go_synchronous(self): """ self._go_synchronous = True - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Metadata Collection - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- @not_mainthread def _refresh_async(self, result_queue, progress_callback=None): @@ -505,7 +506,7 @@ def _refresh(self, progress_callback=None, is_async=False): total = len(function_addresses) start = time.time() - #---------------------------------------------------------------------- + # ---------------------------------------------------------------------- # refresh the core database metadata asynchronously if is_async and self._async_collect_metadata(function_addresses, progress_callback): @@ -516,7 +517,7 @@ def _refresh(self, progress_callback=None, is_async=False): completed = total - len(function_addresses) self._sync_collect_metadata(function_addresses, progress_callback, completed) - #---------------------------------------------------------------------- + # ---------------------------------------------------------------------- end = time.time() logger.debug("Metadata collection took %s seconds" % (end - start)) @@ -526,7 +527,7 @@ def _refresh(self, progress_callback=None, is_async=False): # refresh the internal function/node fast lookup lists self._refresh_lookup() - #---------------------------------------------------------------------- + # ---------------------------------------------------------------------- # reinstall the rename listener hooks now that the refresh is done self._rename_hooks.hook() @@ -637,7 +638,7 @@ def _cache_functions(self, addresses_chunk): # attempt to 'lift' the function from the database try: - function_metadata = FunctionMetadata(address, disassembler_ctx) + function_metadata = FunctionMetadata(address, disassembler_ctx, self.get_function) # # this is not exactly a good thing but it indicates that the @@ -658,9 +659,9 @@ def _cache_functions(self, addresses_chunk): self.nodes.update(function_metadata.nodes) self.functions[address] = function_metadata - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Signal Handlers - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def _name_changed(self, address, new_name): """ @@ -685,9 +686,9 @@ def _name_changed(self, address, new_name): # notify metadata listeners of the rename event self._notify_function_renamed() - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Callbacks - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def metadata_modified(self, callback): """ @@ -725,9 +726,9 @@ def _notify_rebased(self, old_imagebase, new_imagebase): """ notify_callback(self._rebased_callbacks) - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Scheduled - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- @disassembler.execute_read def _scheduled_worker(self): @@ -750,16 +751,39 @@ def _scheduled_worker(self): if self._scheduled_timer: self._scheduled_timer.start(self._scheduled_interval) -#------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ # Function Metadata -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ +class Locker(object): + def __init__(self, is_locked=False): + self.is_locked = is_locked + + def locked(self): + return self.is_locked + + def acquire(self): + self.is_locked = True + + def release(self): + if not self.is_locked: + raise ValueError("can't release not acquired resource") + + self.is_locked = False + + def __enter__(self): + self.acquire() + + def __exit__(self, xc_type, exc_value, exc_traceback): + self.release() + class FunctionMetadata(object): """ Function level metadata cache. """ - def __init__(self, address, disassembler_ctx=None): + def __init__(self, address, disassembler_ctx=None, addr_to_function_metadata=None): # function metadata self.address = address @@ -776,13 +800,57 @@ def __init__(self, address, disassembler_ctx=None): self.instruction_count = 0 self.cyclomatic_complexity = 0 - # collect metdata from the underlying database + self.addr_to_functionMetadata = addr_to_function_metadata + self.called_addresses = [] + self._accumulated_edge_count = None + self._accumulated_instruction_count = None + self.__lock_accumulated_calculation_edge = Locker(False) + self.__lock_accumulated_calculation_instruction = Locker(False) + + # collect metadata from the underlying database self._cache_function(disassembler_ctx) - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Properties - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- + + def __metadata_accumulated_val(self, locker, internal_member_name, external_property_name, initial_value): + if self.__getattribute__(internal_member_name) is not None: + return self.__getattribute__(internal_member_name) + + if self.addr_to_functionMetadata is None: + raise ValueError("can't resolve accumulated_variable_count") + + # There is a loop, return 0 because the real value of the count will be calculated in the locked part. + if locker.locked(): + logger.info("calculating accumulated value for {function_name} reached loop".format(function_name=self.name)) + return 0 + + with locker as _: + accumulated_generic_count = initial_value + for func_addr in self.called_addresses: + metadata_called_func = self.addr_to_functionMetadata(func_addr) + if metadata_called_func is not None: + accumulated_generic_count += metadata_called_func.__getattribute__(external_property_name) + else: + logger.warning("In function {func_addr} code ref {external_func_addr} could not find metadata object" + " for accumulated calculation - can it be external function?".format(func_addr=hex(self.address), external_func_addr=hex(func_addr))) + + self.__setattr__(internal_member_name, accumulated_generic_count) + return self.__getattribute__(internal_member_name) + + @property + def accumulated_edge_count(self): + return self.__metadata_accumulated_val(self.__lock_accumulated_calculation_edge, + "_accumulated_edge_count", + "accumulated_edge_count", initial_value=len(self.edges)) + + @property + def accumulated_instruction_count(self): + return self.__metadata_accumulated_val(self.__lock_accumulated_calculation_edge, + "_accumulated_instruction_count", + "accumulated_instruction_count", initial_value=self.instruction_count) @property def instructions(self): """ @@ -797,9 +865,9 @@ def empty(self): """ return self.size == 0 - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Metadata Population - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def _cache_function(self, disassembler_ctx): """ @@ -817,7 +885,7 @@ def _refresh_nodes(self, disassembler_ctx): """ raise RuntimeError("This function should have been monkey patched...") - def _ida_refresh_nodes(self, _): + def _ida_refresh_nodes(self, disassembler_ctx): """ Refresh function node metadata against an open IDA database. """ @@ -825,7 +893,7 @@ def _ida_refresh_nodes(self, _): function_metadata.nodes = {} # get function & flowchart object from IDA database - function = idaapi.get_func(self.address) + function = idaapi.get_func(self.address) flowchart = idaapi.qflow_chart_t("", function, idaapi.BADADDR, idaapi.BADADDR, 0) # @@ -847,8 +915,9 @@ def _ida_refresh_nodes(self, _): continue # create a new metadata object for this node - node_metadata = NodeMetadata(node.start_ea, node.end_ea, node_id) - + node_code_refs = disassembler_ctx.get_code_refs_from_block(node.start_ea, node.end_ea) + node_metadata = NodeMetadata(node.start_ea, node.end_ea, node_id, addr_code_refs=node_code_refs) + self.called_addresses += node_code_refs # # establish a relationship between this node (basic block) and # this function metadata (its parent) @@ -905,11 +974,12 @@ def _binja_refresh_nodes(self, disassembler_ctx): for i in range(0, count.value): if edges[i].target: - function_metadata.edges[edge_src].append(node._create_instance(BNNewBasicBlockReference(edges[i].target), bv).start) + function_metadata.edges[edge_src].append( + node._create_instance(BNNewBasicBlockReference(edges[i].target), bv).start) core.BNFreeBasicBlockEdgeList(edges, count.value) # NOTE/PERF ~28% of metadata collection time alone... - #for edge in node.outgoing_edges: + # for edge in node.outgoing_edges: # function_metadata.edges[edge_src].append(edge.target.start) def _compute_complexity(self): @@ -959,13 +1029,13 @@ def _compute_complexity(self): # update the map of confirmed (walked) edges confirmed_edges[current_src] = self.edges.pop(current_src) - + # # retain only the 'confirmed' edges. this may differ from the # original edge map because we are only keeping edges that can be # walked from the function entry. (eg, no ida exception handlers) # - + self.edges = confirmed_edges # compute the final cyclomatic complexity for the function @@ -983,9 +1053,16 @@ def _finalize(self): self.instruction_count = sum(node.instruction_count for node in itervalues(self.nodes)) self.cyclomatic_complexity = self._compute_complexity() - #-------------------------------------------------------------------------- + # Clean cache + with self.__lock_accumulated_calculation_edge as _: + self._accumulated_edge_count = None + + with self.__lock_accumulated_calculation_instruction as _: + self._accumulated_instruction_count = None + + # -------------------------------------------------------------------------- # Operator Overloads - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def __eq__(self, other): """ @@ -1000,16 +1077,17 @@ def __eq__(self, other): result &= viewkeys(self.nodes) == viewkeys(other.nodes) return result -#------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ # Node Metadata -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ class NodeMetadata(object): """ Node (basic block) level metadata cache. """ - def __init__(self, start_ea, end_ea, node_id=None, disassembler_ctx=None): + def __init__(self, start_ea, end_ea, node_id=None, disassembler_ctx=None, addr_code_refs=None): # node metadata self.size = end_ea - start_ea @@ -1023,14 +1101,19 @@ def __init__(self, start_ea, end_ea, node_id=None, disassembler_ctx=None): # instruction addresses self.instructions = {} - #---------------------------------------------------------------------- + if addr_code_refs is None: + addr_code_refs = [] + + # list of code refs from this block (to calculate accumulated blocks) + self.addr_code_refs = addr_code_refs + # ---------------------------------------------------------------------- # collect metadata from the underlying database self._cache_node(disassembler_ctx) - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Metadata Population - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def _cache_node(self, disassembler_ctx): """ @@ -1092,15 +1175,15 @@ def _binja_cache_node(self, disassembler_ctx): # save the number of instructions in this block self.instruction_count = len(self.instructions) - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Operator Overloads - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def __str__(self): """ Printable NodeMetadata. """ - output = "" + output = "" output += "Node 0x%08X Info:\n" % self.address output += " Address: 0x%08X\n" % self.address output += " Size: %u\n" % self.size @@ -1130,9 +1213,10 @@ def __eq__(self, other): result &= self.id == other.id return result -#------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ # Async Metadata Helpers -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ @disassembler.execute_ui def metadata_progress(completed, total): @@ -1143,9 +1227,10 @@ def metadata_progress(completed, total): "Collected metadata for %u/%u Functions" % (completed, total) ) -#------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ # MONKEY PATCHING -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ # # We use 'monkey patching' to modify the Metadata class definitions at # runtime. Specifically, we use it to swap in metadata collection routines @@ -1160,6 +1245,7 @@ def metadata_progress(completed, total): if disassembler.NAME == "IDA": import idaapi import idautils + FunctionMetadata._refresh_nodes = FunctionMetadata._ida_refresh_nodes NodeMetadata._cache_node = NodeMetadata._ida_cache_node @@ -1170,6 +1256,7 @@ def metadata_progress(completed, total): import ctypes import binaryninja from binaryninja import core + FunctionMetadata._refresh_nodes = FunctionMetadata._binja_refresh_nodes NodeMetadata._cache_node = NodeMetadata._binja_cache_node diff --git a/plugins/lighthouse/ui/coverage_table.py b/plugins/lighthouse/ui/coverage_table.py index edcee16b..06aa5090 100644 --- a/plugins/lighthouse/ui/coverage_table.py +++ b/plugins/lighthouse/ui/coverage_table.py @@ -13,9 +13,10 @@ logger = logging.getLogger("Lighthouse.UI.Table") -#------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ # CoverageTableView -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ class CoverageTableView(QtWidgets.QTableView): """ @@ -51,8 +52,8 @@ def refresh_theme(self): " outline: none; " "} " + "QHeaderView::section { " - " padding: 1ex;" \ - " margin: 0;" \ + " padding: 1ex;" \ + " margin: 0;" \ "} " + "QTableView::item:selected {" " color: white; " @@ -60,9 +61,9 @@ def refresh_theme(self): "}" ) - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # QTableView Overloads - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def keyPressEvent(self, event): """ @@ -91,9 +92,9 @@ def keyPressEvent(self, event): self.repaint() flush_qt_events() - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Initialization - UI - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def _ui_init(self): """ @@ -136,7 +137,6 @@ def _ui_init_table(self): # set the initial column widths based on their title or contents for i in xrange(self._model.columnCount()): - # determine the pixel width of the column header text title_rect = self._model.headerData(i, QtCore.Qt.Horizontal, QtCore.Qt.SizeHintRole) @@ -145,7 +145,7 @@ def _ui_init_table(self): entry_rect = entry_fm.boundingRect(entry_text) # select the larger of the two potential column widths - column_width = max(title_rect.width(), entry_rect.width()*1.2) + column_width = max(title_rect.width(), entry_rect.width() * 1.2) # save the final column width self.setColumnWidth(i, column_width) @@ -165,8 +165,8 @@ def _ui_init_table(self): vh.hide() # stretch last table column (which is blank) to fill remaining space - #hh.setStretchLastSection(True) - #hh.setCascadingSectionResizes(True) + # hh.setStretchLastSection(True) + # hh.setCascadingSectionResizes(True) hh.setSectionResizeMode(1, QtWidgets.QHeaderView.Stretch) # disable bolding of table column headers when table is selected @@ -190,8 +190,8 @@ def _ui_init_table(self): # specify the fixed pixel height for the table rows # NOTE: don't ask too many questions about this voodoo math :D spacing = entry_fm.height() - entry_fm.xHeight() - tweak = (17*get_dpi_scale() - spacing)/get_dpi_scale() - vh.setDefaultSectionSize(entry_fm.height()+tweak) + tweak = (17 * get_dpi_scale() - spacing) / get_dpi_scale() + vh.setDefaultSectionSize(entry_fm.height() + tweak) def _ui_init_table_ctx_menu_actions(self): """ @@ -237,9 +237,9 @@ def _ui_init_signals(self): hh.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) hh.customContextMenuRequested.connect(self._ui_header_ctx_menu_handler) - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Signal Handlers - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def _ui_entry_double_click(self, index): """ @@ -286,9 +286,9 @@ def _ui_header_ctx_menu_handler(self, position): # process the user action self._process_header_ctx_menu_action(action, column) - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Context Menu (Table Rows) - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def _populate_table_ctx_menu(self): """ @@ -376,9 +376,9 @@ def _process_table_ctx_menu_action(self, action): elif action == self._action_clear_prefix: self._controller.clear_function_prefixes(rows) - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Context Menu (Table Header) - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def _populate_header_ctx_menu(self): """ @@ -403,9 +403,10 @@ def _process_header_ctx_menu_action(self, action, column): if action == self._action_alignment: self._controller.toggle_column_alignment(column) -#------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ # CoverageTableController -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ class CoverageTableController(object): """ @@ -417,9 +418,9 @@ def __init__(self, lctx, model): self._model = model self._last_directory = None - #--------------------------------------------------------------------------- + # --------------------------------------------------------------------------- # Renaming - #--------------------------------------------------------------------------- + # --------------------------------------------------------------------------- @mainthread def rename_table_function(self, row): @@ -436,7 +437,7 @@ def rename_table_function(self, row): "Please enter function name", "Rename Function", original_name - ) + ) # # if the user clicked cancel, or the name they entered @@ -460,7 +461,7 @@ def prefix_table_functions(self, rows): "Please enter a function prefix", "Prefix Function(s)", "MyPrefix" - ) + ) # bail if the user clicked cancel or failed to enter a prefix if not (ok and prefix): @@ -478,9 +479,9 @@ def clear_function_prefixes(self, rows): function_addresses = self._get_function_addresses(rows) disassembler[self.lctx].clear_prefixes(function_addresses) - #--------------------------------------------------------------------------- + # --------------------------------------------------------------------------- # Copy-to-Clipboard - #--------------------------------------------------------------------------- + # --------------------------------------------------------------------------- @mainthread def copy_name(self, rows): @@ -527,9 +528,9 @@ def copy_name_and_address(self, rows): copy_to_clipboard(function_name_and_address.rstrip()) return function_name_and_address - #--------------------------------------------------------------------------- + # --------------------------------------------------------------------------- # Misc - #--------------------------------------------------------------------------- + # --------------------------------------------------------------------------- def navigate_to_function(self, row): """ @@ -596,11 +597,11 @@ def export_to_html(self): # we construct kwargs here for cleaner PySide/PyQt5 compatibility kwargs = \ - { - "filter": "HTML Files (*.html)", - "caption": "Save HTML Report", - "directory": suggested_filepath - } + { + "filter": "HTML Files (*.html)", + "caption": "Save HTML Report", + "directory": suggested_filepath + } # prompt the user with the file dialog, and await their chosen filename(s) filename, _ = file_dialog.getSaveFileName(**kwargs) @@ -616,9 +617,9 @@ def export_to_html(self): lmsg("Saved HTML report to %s" % filename) - #--------------------------------------------------------------------------- + # --------------------------------------------------------------------------- # Internal - #--------------------------------------------------------------------------- + # --------------------------------------------------------------------------- def _get_function_addresses(self, rows): """ @@ -630,9 +631,10 @@ def _get_function_addresses(self, rows): function_addresses.append(address) return function_addresses -#------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ # CoverageTableModel -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ class CoverageTableModel(QtCore.QAbstractTableModel): """ @@ -640,64 +642,74 @@ class CoverageTableModel(QtCore.QAbstractTableModel): """ # named constants for coverage table column indexes - COV_PERCENT = 0 - FUNC_NAME = 1 - FUNC_ADDR = 2 - BLOCKS_HIT = 3 - INST_HIT = 4 - FUNC_SIZE = 5 - COMPLEXITY = 6 + COV_PERCENT = 0 + FUNC_NAME = 1 + FUNC_ADDR = 2 + BLOCKS_HIT = 3 + INST_HIT = 4 + FUNC_SIZE = 5 + COMPLEXITY = 6 + ACCUMULATED_BLOCK_HITS = 7 + ACCUMULATED_INSTRUCTION_HITS = 8 METADATA_ATTRIBUTES = [FUNC_NAME, FUNC_ADDR, FUNC_SIZE, COMPLEXITY] - COVERAGE_ATTRIBUTES = [COV_PERCENT, BLOCKS_HIT, INST_HIT] + COVERAGE_ATTRIBUTES = [COV_PERCENT, BLOCKS_HIT, INST_HIT, ACCUMULATED_BLOCK_HITS, ACCUMULATED_INSTRUCTION_HITS] # column index -> object attribute mapping COLUMN_TO_FIELD = \ - { - COV_PERCENT: "instruction_percent", - FUNC_NAME: "name", - FUNC_ADDR: "address", - BLOCKS_HIT: "nodes_executed", - INST_HIT: "instructions_executed", - FUNC_SIZE: "size", - COMPLEXITY: "cyclomatic_complexity" - } + { + COV_PERCENT: "instruction_percent", + FUNC_NAME: "name", + FUNC_ADDR: "address", + BLOCKS_HIT: "nodes_executed", + INST_HIT: "instructions_executed", + FUNC_SIZE: "size", + COMPLEXITY: "cyclomatic_complexity", + ACCUMULATED_BLOCK_HITS: "accumulated_edge_executed", + ACCUMULATED_INSTRUCTION_HITS: "accumulated_instruction_executed", + } # column headers of the table COLUMN_HEADERS = \ - { - COV_PERCENT: "Cov %", - FUNC_NAME: "Func Name", - FUNC_ADDR: "Address", - BLOCKS_HIT: "Blocks Hit", - INST_HIT: "Instr. Hit", - FUNC_SIZE: "Func Size", - COMPLEXITY: "CC", - } + { + COV_PERCENT: "Cov %", + FUNC_NAME: "Func Name", + FUNC_ADDR: "Address", + BLOCKS_HIT: "Blocks Hit", + INST_HIT: "Instr. Hit", + FUNC_SIZE: "Func Size", + COMPLEXITY: "CC", + ACCUMULATED_BLOCK_HITS: "A-Block Hits", + ACCUMULATED_INSTRUCTION_HITS: "A-Instr. Hits", + } # column header tooltips COLUMN_TOOLTIPS = \ - { - COV_PERCENT: "Coverage Percent", - FUNC_NAME: "Function Name", - FUNC_ADDR: "Function Address", - BLOCKS_HIT: "Number of Basic Blocks Executed", - INST_HIT: "Number of Instructions Executed", - FUNC_SIZE: "Function Size (bytes)", - COMPLEXITY: "Cyclomatic Complexity", - } + { + COV_PERCENT: "Coverage Percent", + FUNC_NAME: "Function Name", + FUNC_ADDR: "Function Address", + BLOCKS_HIT: "Number of Basic Blocks Executed", + INST_HIT: "Number of Instructions Executed", + FUNC_SIZE: "Function Size (bytes)", + COMPLEXITY: "Cyclomatic Complexity", + ACCUMULATED_BLOCK_HITS: "Accumulated number of Basic Blocks Executed", + ACCUMULATED_INSTRUCTION_HITS: "Accumulated number of Instructions Executed", + } # sample column SAMPLE_CONTENTS = \ - [ - " 100.00 ", - " sub_140001B20 ", - " 0x140001b20 ", - " 100 / 100 ", - " 1000 / 1000 ", - " 100000 ", - " 1000 ", - ] + [ + " 100.00 ", + " sub_140001B20 ", + " 0x140001b20 ", + " 100 / 100 ", + " 1000 / 1000 ", + " 100000 ", + " 1000 ", + " 200 / 500 ", + " 5000 / 10000 ", + ] def __init__(self, lctx, parent=None): super(CoverageTableModel, self).__init__(parent) @@ -735,17 +747,17 @@ def __init__(self, lctx, parent=None): self._title_font = QtGui.QFont() self._title_font.setPointSizeF(normalize_to_dpi(10)) - #---------------------------------------------------------------------- + # ---------------------------------------------------------------------- # Sorting - #---------------------------------------------------------------------- + # ---------------------------------------------------------------------- # attributes to track the model's last known (column) sort state self._last_sort = self.FUNC_ADDR self._last_sort_order = QtCore.Qt.AscendingOrder - #---------------------------------------------------------------------- + # ---------------------------------------------------------------------- # Filters - #---------------------------------------------------------------------- + # ---------------------------------------------------------------------- # OPTION: display 0% coverage entries self._hide_zero = False @@ -753,9 +765,9 @@ def __init__(self, lctx, parent=None): # OPTION: display functions matching search_string (substring) self._search_string = "" - #---------------------------------------------------------------------- + # ---------------------------------------------------------------------- # Signals - #---------------------------------------------------------------------- + # ---------------------------------------------------------------------- # register for cues from the director self._director.coverage_switched(self._internal_refresh) @@ -771,9 +783,9 @@ def refresh_theme(self): self._blank_coverage.coverage_color = self.lctx.palette.table_coverage_none self._data_changed() - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # QAbstractTableModel Overloads - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def flags(self, index): return QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsSelectable @@ -818,7 +830,7 @@ def headerData(self, column, orientation, role=QtCore.Qt.DisplayRole): if role == QtCore.Qt.SizeHintRole: title_fm = QtGui.QFontMetricsF(self._title_font) title_rect = title_fm.boundingRect(self.COLUMN_HEADERS[column]) - padded = QtCore.QSize(int(title_rect.width()*1.45), int(title_rect.height()*1.75)) + padded = QtCore.QSize(int(title_rect.width() * 1.45), int(title_rect.height() * 1.75)) return padded # unhandeled header request @@ -837,7 +849,7 @@ def data(self, index, role=QtCore.Qt.DisplayRole): # lookup the function info for this row try: - function_address = self.row2func[index.row()] + function_address = self.row2func[index.row()] function_metadata = self.lctx.metadata.functions[function_address] # @@ -871,7 +883,7 @@ def data(self, index, role=QtCore.Qt.DisplayRole): # Coverage % - (by instruction execution) if column == self.COV_PERCENT: - return "%5.2f" % (function_coverage.instruction_percent*100) + return "%5.2f" % (function_coverage.instruction_percent * 100) # Function Name elif column == self.FUNC_NAME: @@ -899,9 +911,19 @@ def data(self, index, role=QtCore.Qt.DisplayRole): elif column == self.COMPLEXITY: return "%u" % function_metadata.cyclomatic_complexity + elif column == self.ACCUMULATED_BLOCK_HITS: + return "{accumulated_block_executed} / {accumulated_blocks}".format( + accumulated_block_executed=function_coverage.accumulated_edge_executed, + accumulated_blocks=function_metadata.accumulated_edge_count) + + elif column == self.ACCUMULATED_INSTRUCTION_HITS: + return "{accumulated_instructions_executed} / {accumulated_instructions}".format( + accumulated_instructions_executed=function_coverage.accumulated_instruction_executed, + accumulated_instructions=function_metadata.accumulated_instruction_count) + # cell background color request elif role == QtCore.Qt.BackgroundRole: - function_address = self.row2func[index.row()] + function_address = self.row2func[index.row()] function_coverage = self._director.coverage.functions.get( function_address, self._blank_coverage @@ -919,9 +941,9 @@ def data(self, index, role=QtCore.Qt.DisplayRole): # unhandeled request, nothing to do return None - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Sorting - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def sort(self, column, sort_order): """ @@ -1001,9 +1023,9 @@ def sort(self, column, sort_order): self._last_sort = column self._last_sort_order = sort_order - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Public - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def set_column_alignment(self, column, alignment): """ @@ -1030,11 +1052,11 @@ def get_modeled_coverage_percent(self): ) # compute coverage percentage of the visible functions - return (float(instructions_executed) / (instruction_count or 1))*100 + return (float(instructions_executed) / (instruction_count or 1)) * 100 - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # HTML Export - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def to_html(self): """ @@ -1052,7 +1074,7 @@ def to_html(self): body_elements = [summary_html, table_html] body_html = "%s" % '\n'.join(body_elements) body_css = \ - """ + """ body {{ font-family: Arial, Helvetica, sans-serif; @@ -1060,9 +1082,9 @@ def to_html(self): background-color: {page_bg}; }} """.format( - page_fg=palette.table_text.name(), - page_bg=palette.html_page_background.name() - ) + page_fg=palette.table_text.name(), + page_bg=palette.html_page_background.name() + ) # HTML tag css_elements = [body_css, summary_css, table_css] @@ -1088,21 +1110,21 @@ def _generate_html_summary(self): title_html = "

Lighthouse Coverage Report

" # summary details - detail = lambda x,y: '
  • %s: %s
  • ' % (x,y) - database_percent = coverage.instruction_percent*100 + detail = lambda x, y: '
  • %s: %s
  • ' % (x, y) + database_percent = coverage.instruction_percent * 100 table_percent = self.get_modeled_coverage_percent() details = \ - [ - detail("Target Binary", metadata.filename), - detail("Coverage Name", coverage.name), - detail("Coverage File", coverage.filepath), - detail("Database Coverage", "%1.2f%%" % database_percent), - detail("Table Coverage", "%1.2f%%" % table_percent), - detail("Timestamp", time.ctime()), - ] + [ + detail("Target Binary", metadata.filename), + detail("Coverage Name", coverage.name), + detail("Coverage File", coverage.filepath), + detail("Database Coverage", "%1.2f%%" % database_percent), + detail("Table Coverage", "%1.2f%%" % table_percent), + detail("Timestamp", time.ctime()), + ] list_html = "" % '\n'.join(details) list_css = \ - """ + """ .detail {{ font-weight: bold; color: {page_fg}; @@ -1111,9 +1133,9 @@ def _generate_html_summary(self): color: {detail_fg}; }} """.format( - page_fg=palette.table_text.name(), - detail_fg=palette.html_summary_text.name() - ) + page_fg=palette.table_text.name(), + detail_fg=palette.html_summary_text.name() + ) # title + summary summary_html = title_html + list_html @@ -1129,7 +1151,7 @@ def _generate_html_table(self): # generate the table's column title row header_cells = [] - for i in xrange(self.columnCount()-1): + for i in xrange(self.columnCount() - 1): header_cells.append( "%s" % self.headerData(i, QtCore.Qt.Horizontal) ) @@ -1138,7 +1160,7 @@ def _generate_html_table(self): # generate the table's coverage rows for row in xrange(self.rowCount()): row_cells = [] - for column in xrange(self.columnCount()-1): + for column in xrange(self.columnCount() - 1): index = self.index(row, column) row_cells.append("%s" % self.data(index)) row_color = self.data(index, QtCore.Qt.BackgroundRole).name() @@ -1153,7 +1175,7 @@ def _generate_html_table(self): # generate the final HTML table table_html = "%s
    " % '\n'.join(html_rows) table_css = \ - """ + """ table {{ text-align: center; white-space: pre; @@ -1181,15 +1203,15 @@ def _generate_html_table(self): padding: 1ex 1em 1ex 1em; }} """.format( - table_bg=palette.table_background.name(), - table_fg=palette.table_text.name() - ) + table_bg=palette.table_background.name(), + table_fg=palette.table_text.name() + ) return (table_html, table_css) - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Filters - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def filter_zero_coverage(self, hide): """ @@ -1217,9 +1239,9 @@ def filter_string(self, search_string): self._search_string = search_string self._internal_refresh() - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Refresh - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def refresh(self): """ @@ -1274,9 +1296,9 @@ def _refresh_data(self): # loop through *all* the functions as defined in the active metadata for function_address in metadata.functions: - #------------------------------------------------------------------ + # ------------------------------------------------------------------ # Filters - START - #------------------------------------------------------------------ + # ------------------------------------------------------------------ # OPTION: ignore items with 0% coverage items if self._hide_zero and not function_address in coverage.functions: @@ -1286,9 +1308,9 @@ def _refresh_data(self): if not self._search_string in normalize(metadata.functions[function_address].name): continue - #------------------------------------------------------------------ + # ------------------------------------------------------------------ # Filters - END - #------------------------------------------------------------------ + # ------------------------------------------------------------------ # store a reference to the listed function's metadata self._visible_metadata[function_address] = metadata.functions[function_address] @@ -1311,9 +1333,9 @@ def _refresh_data(self): # bake the final number of rows into the model self._row_count = len(self.row2func) - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # Qt Notifications - #-------------------------------------------------------------------------- + # -------------------------------------------------------------------------- @disassembler.execute_ui def _data_changed(self): diff --git a/plugins/lighthouse/util/disassembler/api.py b/plugins/lighthouse/util/disassembler/api.py index 4c4f6855..28f4d8e1 100644 --- a/plugins/lighthouse/util/disassembler/api.py +++ b/plugins/lighthouse/util/disassembler/api.py @@ -357,6 +357,9 @@ def set_function_name_at(self, function_address, new_name): """ pass + @abc.abstractmethod + def get_code_refs_from_block(self, start_addr, end_addr): + pass #-------------------------------------------------------------------------- # Hooks API #-------------------------------------------------------------------------- diff --git a/plugins/lighthouse/util/disassembler/ida_api.py b/plugins/lighthouse/util/disassembler/ida_api.py index 46adb893..14707447 100644 --- a/plugins/lighthouse/util/disassembler/ida_api.py +++ b/plugins/lighthouse/util/disassembler/ida_api.py @@ -325,6 +325,7 @@ class IDAContextAPI(DisassemblerContextAPI): def __init__(self, dctx): super(IDAContextAPI, self).__init__(dctx) + self.__external_functions_addresses = None @property def busy(self): @@ -365,6 +366,45 @@ def navigate_to_function(self, function_address, address): def set_function_name_at(self, function_address, new_name): idaapi.set_name(function_address, new_name, idaapi.SN_NOWARN) + def is_function_is_external(self, address): + def add_external_func_cb(ea, _, __): + self.__external_functions_addresses.add(ea) + # True for continue enumeration + return True + + # static function variable that should be calculated only once. + if self.__external_functions_addresses is None: + self.__external_functions_addresses = set() # optimizations, in list it will take a lot of time + number_of_imports = idaapi.get_import_module_qty() + for i in range(number_of_imports): + name = idaapi.get_import_module_name(i) + if not name: + logger.info("Failed to get import module name for #%d" % i) + continue + + idaapi.enum_import_names(i, add_external_func_cb) + + if address in self.__external_functions_addresses: + return True + + return False + + def get_code_refs_from_block(self, start_addr, end_addr): + code_refs_from_block = [] + for current_addr in idautils.Heads(start_addr, end_addr): + for result in idautils.XrefsFrom(current_addr, idaapi.fl_CN): + type_xref = idautils.XrefTypeName(result.type).lower() + # to ignore Jumps + if "code" not in type_xref or "call" not in type_xref: + continue + + addr_call_to = result.to + if self.is_function_is_external(addr_call_to): # external references outside the PE are ignored + continue + + code_refs_from_block.append(addr_call_to) + + return code_refs_from_block #-------------------------------------------------------------------------- # Hooks API #--------------------------------------------------------------------------