int_maketodo.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2017-2020  The Project X-Ray Authors.
#
# Use of this source code is governed by a ISC-style
# license that can be found in the LICENSE file or at
# https://opensource.org/licenses/ISC
#
# SPDX-License-Identifier: ISC

import os, re
from prjxray import util


def noprefix(tag):
    n = tag.find('.')
    assert n > 0
    return tag[n + 1:]


def getprefix(tag):
    n = tag.find('.')
    assert n > 0
    return tag[0:n]


def load_pipfile(pipfile, verbose=False):
    '''Returns a set of tags containing real tile type prefixes (ex: INT_L)'''
    todos = set()
    tile_type = None
    with open(pipfile, "r") as f:
        # INT_L.WW2BEG0.SR1BEG_S0
        for line in f:
            tag = line.strip().split(' ')[0]
            prefix_line = getprefix(tag)
            if tile_type is None:
                tile_type = prefix_line
            else:
                assert tile_type == prefix_line
            todos.add(tag)
    return todos, tile_type


def balance_todo_list(
        pipfile,
        todos,
        balance_wire_re,
        balance_wire_direction,
        balance_wire_cnt,
        verbose=False):
    """Balance the contents of the todo list

    Todo list balancing allows to specify the name, direction and minimal number of
    occurrences of a PIP wire in the final todo list.
    The mechanism should be used in cases where a fuzzer times out because of an
    unsolvable todo list, i.e. the netlist and resulting segdata generated from an
    iteration keep segmatch from properly disambiguating the bits for some features.

    When the balance wire name regexp is specified it's guaranteed that all PIPs
    with matching wire name (whether we want to match a src or dst wire has to be
    specified with the --balance-wire-direction switch) will have at least the number
    of entries specified with the --balance-wire-cnt switch in the final todo list.
    """
    orig_todos, tile_type = load_pipfile(pipfile, verbose=verbose)
    if balance_wire_re is not None:
        todo_wires = {}
        verbose and print("Start balancing the TODO list")
        for todo in todos:
            tile_type, dst, src = todo.split(".")
            wire = src
            other_wire = dst
            if balance_wire_direction not in "src":
                wire = dst
                other_wire = src
            balance_wire_match = re.match(balance_wire_re, wire)
            if balance_wire_match is None:
                continue
            if wire not in todo_wires:
                todo_wires[wire] = set()
            todo_wires[wire].add(other_wire)
        for wire, other_wires in todo_wires.items():
            if len(other_wires) >= balance_wire_cnt:
                continue
            else:
                for todo in orig_todos:
                    tile_type, dst, src = todo.split(".")
                    if balance_wire_direction in "src":
                        if wire in src and dst not in todo_wires[wire]:
                            todo_wires[wire].add(dst)
                    else:
                        if wire in dst and src not in todo_wires[wire]:
                            todo_wires[wire].add(src)
                    if len(todo_wires[wire]) == balance_wire_cnt:
                        break
        for wire, other_wires in todo_wires.items():
            if len(other_wires) < balance_wire_cnt:
                verbose and print(
                    "Warning: failed to balance the todo list for wire {}, there are only {} PIPs which meet the requirement: {}"
                    .format(wire, len(other_wires), other_wires))
            for other_wire in other_wires:
                line = tile_type + "."
                if balance_wire_direction in "src":
                    line += other_wire + "." + wire
                else:
                    line += wire + "." + other_wire
                verbose and print("Adding {}".format(line))
                todos.add(line)
        verbose and print("Finished balancing the TODO list")


def maketodo(
        pipfile,
        dbfile,
        intre,
        exclude_re=None,
        balance_wire_re=None,
        balance_wire_direction=None,
        balance_wire_cnt=None,
        not_endswith=None,
        verbose=False):
    '''
    db files start with INT., but pipfile lines start with INT_L
    Normalize by removing before the first dot
    050-intpips doesn't care about contents, but most fuzzers use the tile type prefix
    '''

    todos, tile_type = load_pipfile(pipfile, verbose=verbose)
    verbose and print('%s: %u entries' % (pipfile, len(todos)))
    if not todos:
        verbose and print('%s: %u entries, done!' % (pipfile, len(todos)))
        return

    verbose and print("pipfile todo sample: %s" % list(todos)[0])

    if 0 and verbose:
        print("TODOs")
        for todo in sorted(list(todos)):
            print('  %s' % todo)

    verbose and print('Pre db %s: %u entries' % (dbfile, len(todos)))
    # Allow against empty db
    if os.path.exists(dbfile):
        verbose and print("Loading %s" % dbfile)
        with open(dbfile, "r") as f:
            # INT.BYP_ALT0.BYP_BOUNCE_N3_3 !22_07 !23_07 !25_07 21_07 24_07
            for line in f:
                tag, _bits, mode, _ = util.parse_db_line(line.strip())
                # Only count resolved entries
                if mode:
                    continue
                # INT.BLAH => INT_L.BLAH
                tag = tile_type + '.' + noprefix(tag)

                # bipips works on a subset
                if tag in todos:
                    todos.remove(tag)
                else:
                    verbose and print(
                        "WARNING: couldnt remove %s (line %s)" %
                        (tag, line.strip()))
    else:
        verbose and print("WARNING: dbfile doesnt exist: %s" % dbfile)
    verbose and print('Post db %s: %u entries' % (dbfile, len(todos)))

    drops = 0
    lines = 0
    filtered_todos = set()
    for line in todos:
        include = re.match(intre, line) is not None

        if include and not_endswith is not None:
            include = not line.endswith(not_endswith)

        if include and exclude_re is not None:
            include = re.match(exclude_re, line) is None

        if include:
            filtered_todos.add(line)
        else:
            drops += 1
        lines += 1
    verbose and print('Print %u entries w/ %u drops' % (lines, drops))

    balance_todo_list(
        pipfile, filtered_todos, balance_wire_re, balance_wire_direction,
        balance_wire_cnt, verbose)
    for todo in filtered_todos:
        print(todo)


def run(
        build_dir,
        db_dir,
        pip_dir,
        intre,
        sides,
        l,
        r,
        pip_type,
        seg_type,
        exclude_re=None,
        balance_wire_re=None,
        balance_wire_direction=None,
        balance_wire_cnt=None,
        not_endswith=None,
        verbose=False):
    if db_dir is None:
        db_dir = "%s/%s" % (
            os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE"))

    if pip_dir is None:
        pip_dir = "%s/piplist/build/%s" % (
            os.getenv("XRAY_FUZZERS_DIR"), pip_type)

    assert intre, "RE is required"

    for side in sides:
        if side == "l" and not l:
            continue

        if side == "r" and not r:
            continue

        if side == "xl":
            segfile = "l{}".format(seg_type)
            pipfile = "l{}".format(pip_type)
        elif side == "xr":
            segfile = "r{}".format(seg_type)
            pipfile = "r{}".format(pip_type)
        elif side != "":
            segfile = "{}_{}".format(seg_type, side)
            pipfile = "{}_{}".format(pip_type, side)
        else:
            segfile = "{}".format(seg_type)
            pipfile = "{}".format(pip_type)

        maketodo(
            "%s/%s.txt" % (pip_dir, pipfile),
            "%s/segbits_%s.db" % (db_dir, segfile),
            intre,
            exclude_re=exclude_re,
            balance_wire_re=balance_wire_re,
            balance_wire_direction=balance_wire_direction,
            balance_wire_cnt=balance_wire_cnt,
            not_endswith=not_endswith,
            verbose=verbose)


def main():
    import argparse

    parser = argparse.ArgumentParser(
        description="Print list of known but unsolved PIPs")

    parser.add_argument('--verbose', action='store_true', help='')
    parser.add_argument('--build-dir', default="build", help='')
    parser.add_argument('--db-dir', default=None, help='')
    parser.add_argument('--pip-dir', default=None, help='')
    parser.add_argument('--re', required=True, help='')
    parser.add_argument('--exclude-re', required=False, default=None, help='')
    parser.add_argument(
        '--balance-wire-re', required=False, default=None, help='')
    parser.add_argument(
        '--balance-wire-direction', required=False, default="src", help='')
    parser.add_argument(
        '--balance-wire-cnt', required=False, default="1", help='')
    parser.add_argument(
        '--balance-re-wire', required=False, default="src", help='')
    parser.add_argument('--pip-type', default="pips_int", help='')
    parser.add_argument('--seg-type', default="int", help='')
    parser.add_argument('--sides', default="l,r", help='')
    util.add_bool_arg(parser, '--l', default=True, help='')
    util.add_bool_arg(parser, '--r', default=True, help='')
    parser.add_argument(
        '--not-endswith', help='Drop lines if they end with this')
    args = parser.parse_args()

    run(
        build_dir=args.build_dir,
        db_dir=args.db_dir,
        pip_dir=args.pip_dir,
        intre=args.re,
        exclude_re=args.exclude_re,
        balance_wire_re=args.balance_wire_re,
        balance_wire_direction=args.balance_wire_direction,
        balance_wire_cnt=int(args.balance_wire_cnt),
        sides=args.sides.split(','),
        l=args.l,
        r=args.r,
        pip_type=args.pip_type,
        seg_type=args.seg_type,
        not_endswith=args.not_endswith,
        verbose=args.verbose)


if __name__ == '__main__':
    main()