04_eval_data/coverage/bb_mann_whitney.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2aebc5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import dateutil.parser as dp\n",
    "import sys\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import rcParams\n",
    "import matplotlib\n",
    "import os\n",
    "import glob\n",
    "from collections import OrderedDict\n",
    "import numpy as np\n",
    "import scipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49122bad",
   "metadata": {},
   "outputs": [],
   "source": [
    "frameworks = ['safire', 'halucinator', 'libafl_halucinator', 'fuzzware_hal', 'fuzzware_nohal',  ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "454d8903",
   "metadata": {},
   "outputs": [],
   "source": [
    "targets = ['atmel_6lowpan_udp_rx', 'atmel_6lowpan_udp_tx', 'nxp_lwip_http', 'samr21_http', 'p2im_plc',  'p2im_drone', 'st-plc', 'wycinwyc', 'stm32_tcp_echo_client', 'stm32_tcp_echo_server', 'stm32_udp_echo_client', 'stm32_udp_echo_server']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd4ac703",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_duplicates(data):\n",
    "    data_in = OrderedDict(data)\n",
    "    data_out = {}\n",
    "    for k, v in data_in.items():\n",
    "        if v not in data_out.values():\n",
    "            data_out[k] = v\n",
    "    return OrderedDict(data_out)\n",
    "\n",
    "def parse_fuzzware_data(data):\n",
    "    data = data.split('\\n')[1:]\n",
    "    data = [(d.split('\\t')[:2]) for d in data if d != '']\n",
    "    data = {int(d[0]): int(d[1]) for d in data}\n",
    "    return remove_duplicates(data)\n",
    "\n",
    "def parse_safire_data(data):\n",
    "    data = OrderedDict(eval(data))\n",
    "    t0 = next(iter(data.keys()))\n",
    "    t0 = int(dp.parse(t0).timestamp())\n",
    "    data = {int(dp.parse(k).timestamp())-t0:v for k,v in data.items()}\n",
    "    return remove_duplicates(data)\n",
    "    \n",
    "\n",
    "def parse_halucinator_data(data):\n",
    "    data = eval(data)\n",
    "    data = {int(k):v for k,v in data.items()}\n",
    "    data = OrderedDict(data)\n",
    "    t0 = next(iter(data.keys()))\n",
    "    data = {k-t0:v for k,v in data.items()}\n",
    "    return remove_duplicates(data)\n",
    "\n",
    "# readall data\n",
    "all_data = {}\n",
    "for t in targets:\n",
    "    all_data[t] = {}\n",
    "    for fw in frameworks:\n",
    "        print(f\"Processing {t} for {fw}\")\n",
    "\n",
    "        all_data[t][fw] = {}\n",
    "        files = glob.glob(f'{fw}_data/{fw}_{t}*.data')\n",
    "        assert(len(files) == 5)\n",
    "        for i in range(5):\n",
    "            with open(files[i], 'r') as f:\n",
    "                data = f.read()\n",
    "            if 'fuzzware' in fw:\n",
    "                data = parse_fuzzware_data(data)\n",
    "            elif 'safire' in fw or 'libafl_halucinator' in fw:\n",
    "                data = parse_safire_data(data)\n",
    "            elif 'halucinator' in fw:\n",
    "                data = parse_halucinator_data(data)\n",
    "            else:\n",
    "                assert(\"NOT REACHABLE\")\n",
    "            all_data[t][fw][f'run-{i}'] = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7aaa512",
   "metadata": {},
   "outputs": [],
   "source": [
    "for fw in frameworks:\n",
    "    if fw == 'safire':\n",
    "        continue\n",
    "    found_violation = False\n",
    "    print(f'=== Comparing SafireFuzz with {fw} ===')\n",
    "    for t in targets:\n",
    "\n",
    "        #if fw != 'fuzzware_hal':\n",
    "        #    continue\n",
    "\n",
    "        safire_maxs = [max(all_data[t]['safire'][f'run-{i}'].values()) for i in range(5) ]\n",
    "        target_maxs = [max(all_data[t][fw][f'run-{i}'].values()) for i in range(5) ]\n",
    "        mannu = scipy.stats.mannwhitneyu(safire_maxs, target_maxs, method='auto')\n",
    "        if mannu.pvalue > 0.05:\n",
    "            print(f'Offending P-Value while comparing against {t}: {mannu.pvalue:.03f}')\n",
    "            found_violation=True\n",
    "    if found_violation is False:\n",
    "        print('No offending P-Values found')\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95307a20",
   "metadata": {},
   "outputs": [],
   "source": [
    "for fw in frameworks:\n",
    "    print(f'=== Calculating median and stdev for {fw} experiments ===')\n",
    "\n",
    "    for t in targets:\n",
    "\n",
    "        target_maxs = [max(all_data[t][fw][f'run-{i}'].values()) for i in range(5) ]\n",
    "        median = sorted(target_maxs)[2]\n",
    "        #print(sorted(target_maxs))\n",
    "        print(f'Sample: {t} - Median: {median} Stddev: {np.std(target_maxs)}')\n",
    "    print()\n",
    "            \n",
    "        "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}