synthetic_networks_experiments.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4VdiGhQd57n1"
   },
   "source": [
    "Code to (re)produce results in the paper \n",
    "\"Manipulating the Online Marketplace of Ideas\" \n",
    "by Xiaodan Lou, Alessandro Flammini, and Filippo Menczer\n",
    "https://arxiv.org/abs/1907.06130\n",
    "\n",
    "Notes:\n",
    "* Need Python 3.6 or later; eg: `module load python/3.6.6`\n",
    "* Remember link direction is following, opposite of info spread!\n",
    "* For large `n_humans`, it's much faster to run the simulations in parallel on a server or cluster, eg, one process for each combination of parameters (gamma, phi, mu...)\n",
    "\n",
    "Parameters and default values:\n",
    "```\n",
    "n_humans = 1000 # 10k for paper\n",
    "beta = 0.1 # bots/humans ratio; 0.1 for paper\n",
    "p = 0.5 # for network clustering; 0.5 for paper\n",
    "k_out = 3 # average no. friends within humans & bots; 3 for paper\n",
    "alpha = 15 # depth of feed; 15 for paper\n",
    "mu = 0.5 # average prob of new meme vs retweet; 0.75 for earlier draft\n",
    "phi = 1 # bot deception >= 1: meme fitness higher than quality\n",
    "gamma = 0.1 # infiltration: probability that a human follows each bot\n",
    "epsilon = 0.01 # threshold used to check for steady-state convergence\n",
    "n_runs = 10 # number of simulations to average results\n",
    "csvfile = 'results.csv' # to save results for plotting\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-25T15:21:42.454586Z",
     "start_time": "2020-05-25T15:21:40.763176Z"
    },
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "byMDogYTqly4"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import networkx as nx\n",
    "import random\n",
    "import numpy\n",
    "import numpy as np\n",
    "import math\n",
    "import statistics\n",
    "import csv\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import stats\n",
    "from operator import itemgetter\n",
    "from collections import defaultdict\n",
    "import sys\n",
    "import fcntl\n",
    "import time\n",
    "import pickle\n",
    "import bot_model\n",
    "\n",
    "%matplotlib inline\n",
    "assert(nx.__version__ >= '2.4')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "iqvOoIzuIJGY"
   },
   "source": [
    "# Loop over parameters and simulation runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def main_exp(preferential_targeting_flag, phis, gammas):\n",
    "    \n",
    "    condition = \"prefer\" if preferential_targeting_flag else \"random\"\n",
    "    save_dir = \"results/\" + condition\n",
    "    if not os.path.exists(save_dir):\n",
    "        os.makedirs(save_dir)\n",
    "\n",
    "    for phi in phis:\n",
    "      for gamma in gammas:\n",
    "        \n",
    "        valid_tracked_memes_all = []\n",
    "        bad_memes_selected_time_all = {}\n",
    "        avg_quality_all = []\n",
    "        avg_diversity_all = []\n",
    "        \n",
    "        for sim in range(n_runs):\n",
    "          print('Running Simulation ', sim, ' for phi = ', phi, ', gamma = ', gamma, ' ...', flush=True)\n",
    "\n",
    "          # simulation start\n",
    "          q, q_net = bot_model.simulation(preferential_targeting_flag, return_net=True, count_forgotten=True, track_meme=True, phi=phi, gamma=gamma) \n",
    "\n",
    "          #### statistic current nth-run data ####\n",
    "\n",
    "          ## avg quality ##\n",
    "          avg_quality_all.append(q)\n",
    "          ## end avg quality ##\n",
    "\n",
    "          ## tracked memes ##\n",
    "          for meme in bot_model.track_memes.popularity:\n",
    "            valid_tracked_memes_all.append((meme[0], bot_model.track_memes.popularity[meme]))\n",
    "          ## end tracked memes ##\n",
    "\n",
    "          ## bad memes selected ##\n",
    "          for meme_id, selected_time in bot_model.track_memes.bad_popularity.items():\n",
    "            if meme_id not in bad_memes_selected_time_all:\n",
    "              bad_memes_selected_time_all[meme_id] = [0, 0]\n",
    "            bad_memes_selected_time_all[meme_id][0] += selected_time[0]\n",
    "            bad_memes_selected_time_all[meme_id][1] += selected_time[1]\n",
    "          ## end bad memes selected ##\n",
    "\n",
    "          ## avg diversity ##\n",
    "          for agent in q_net.nodes:\n",
    "            qualities = []\n",
    "            for m in q_net.nodes[agent]['feed']:\n",
    "              qualities.append(m[0])\n",
    "            unique_qua, unique_qua_cnt = np.unique(qualities, return_counts=True)\n",
    "            portion_of_qua = unique_qua_cnt / np.sum(unique_qua_cnt)\n",
    "            diversity = - np.sum(portion_of_qua * np.log(portion_of_qua))\n",
    "            avg_diversity_all.append(diversity)\n",
    "            # could in theory calculate diversity based on fitness...\n",
    "          ## end avg diversity ##\n",
    "\n",
    "          #### end statistic current nth-run data ####\n",
    "\n",
    "        # average the counts of bad meme selections across runs\n",
    "        for meme_id, selected_time in bad_memes_selected_time_all.items():\n",
    "          bad_memes_selected_time_all[meme_id][0] /= n_runs\n",
    "          bad_memes_selected_time_all[meme_id][1] /= n_runs\n",
    "\n",
    "        # save avg quality\n",
    "        fp = open(\"{}/avg_quality_{}_phi{}_gamma{}.pkl\".format(save_dir, condition, phi, gamma), \"wb\")\n",
    "        pickle.dump(np.mean(avg_quality_all), fp)\n",
    "        fp.close()\n",
    "\n",
    "        # save tracked memes\n",
    "        fp = open(\"{}/tracked_memes_{}_phi{}_gamma{}.pkl\".format(save_dir, condition, phi, gamma), \"wb\")\n",
    "        pickle.dump(valid_tracked_memes_all, fp)\n",
    "        fp.close()\n",
    "\n",
    "        # save bad meme selected times\n",
    "        fp = open(\"{}/bad_memes_selected_time_{}_phi{}_gamma{}.pkl\".format(save_dir, condition, phi, gamma), \"wb\")\n",
    "        pickle.dump(bad_memes_selected_time_all, fp)\n",
    "        fp.close()\n",
    "\n",
    "        # save avg diversity\n",
    "        fp = open(\"{}/avg_diversity_{}_phi{}_gamma{}.pkl\".format(save_dir, condition, phi, gamma), \"wb\")\n",
    "        pickle.dump(np.mean(avg_diversity_all), fp)\n",
    "        fp.close()\n",
    "\n",
    "        # save kendall\n",
    "        quality, number_selected = zip(*valid_tracked_memes_all)\n",
    "        kendall_tau, _ = stats.kendalltau(quality, number_selected)\n",
    "        fp = open(\"{}/kendall_{}_phi{}_gamma{}.pkl\".format(save_dir, condition, phi, gamma), \"wb\")\n",
    "        pickle.dump(kendall_tau, fp)\n",
    "        fp.close()  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Main experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "phis = [1, 5, 10] \n",
    "gammas = [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0] \n",
    "main_exp(False, phis, gammas)\n",
    "#main_exp(True, phis, gammas)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "PDTP02u76ZmW"
   },
   "source": [
    "# Plots for figures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "condition = \"random\"\n",
    "save_dir = \"results/\" + condition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-25T15:25:56.755433Z",
     "start_time": "2020-05-25T15:25:54.671993Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Main plots of average quality, diversity, and Kendall tau as functions of gamma and phi params\n",
    "\n",
    "xs = phis\n",
    "ys = gammas\n",
    "phis1 = phis\n",
    "phis2 = phis\n",
    "wires = gammas\n",
    "new_wires = gammas\n",
    "cmap = None\n",
    "xlabel = '$\\\\phi$'\n",
    "ylabel = '$\\\\gamma$'\n",
    "\n",
    "kendall_pic_title = 'Discriminative power'\n",
    "avg_quality_pic_title = 'Average Quality'\n",
    "diversity_pic_title = 'Diversity'\n",
    "\n",
    "figure = plt.figure(figsize=(13, 15), facecolor='w')\n",
    "markers = [\"o\", \"s\", \"^\"]\n",
    "\n",
    "### 1. average quality ###\n",
    "file_template = \"{}/avg_quality_{}_phi{}_gamma{}.pkl\"\n",
    "\n",
    "# distr plot\n",
    "ax = figure.add_subplot(3,2,1)\n",
    "for idx, phi in enumerate(phis1):\n",
    "    avg_qualities = []\n",
    "    stds = []\n",
    "    for gamma in wires:\n",
    "        fname = file_template.format(save_dir, condition, phi, gamma)\n",
    "        fp = open(fname, \"rb\")\n",
    "        data = pickle.load(fp)\n",
    "        fp.close()\n",
    "        avg_qualities.append(np.mean(data))\n",
    "        stds.append(np.std(data))\n",
    "    ax.plot(new_wires, avg_qualities, marker=markers[idx], label='$\\\\phi$:'+str(h))\n",
    "\n",
    "ax.set_xlabel('$\\\\gamma$', fontsize=14)\n",
    "ax.set_ylabel('Average quality', fontsize=14)\n",
    "ax.set_xscale('log')\n",
    "ax.set_xlim((new_wires[0], new_wires[-1]))\n",
    "ax.set_xlim((new_wires[0], new_wires[-1]))\n",
    "ax.legend(loc='upper right', fontsize=14)\n",
    "\n",
    "# heatmap plot\n",
    "ax = figure.add_subplot(3,2,2)\n",
    "grid = np.zeros((len(wires), len(phis2)))\n",
    "for i, gamma in enumerate(wires):\n",
    "    for j, phi in enumerate(phis2):\n",
    "        fname = file_template.format(save_dir, condition, phi, gamma)\n",
    "        fp = open(fname, \"rb\")\n",
    "        data = pickle.load(fp)\n",
    "        fp.close()\n",
    "        grid[i, j] = np.mean(data)\n",
    "bot_model.draw_heatmap(ax, grid, xs, ys, xlabel, ylabel, cmap, avg_quality_pic_title)\n",
    "\n",
    "\n",
    "### 2. average diversity ###\n",
    "file_template = \"{}/avg_diversity_{}_phi{}_gamma{}.pkl\"\n",
    "\n",
    "# distr plot\n",
    "ax = figure.add_subplot(3,2,3)\n",
    "for idx, phi in enumerate(phis1):\n",
    "    avg_diversities = []\n",
    "    stds = []\n",
    "    for gamma in wires:\n",
    "        fname = file_template.format(save_dir, condition, phi, gamma)\n",
    "        fp = open(fname, \"rb\")\n",
    "        data = pickle.load(fp)\n",
    "        fp.close()\n",
    "        avg_diversities.append(np.mean(data))\n",
    "        stds.append(np.std(data))\n",
    "    ax.plot(new_wires, avg_diversities, marker=markers[idx], label='$\\\\phi$:'+str(h))\n",
    "\n",
    "ax.set_xlabel('$\\\\gamma$', fontsize=14)\n",
    "ax.set_ylabel('Diversity', fontsize=14)\n",
    "ax.set_xscale('log')\n",
    "ax.set_xlim((new_wires[0], new_wires[-1]))\n",
    "ax.set_xlim((new_wires[0], new_wires[-1]))\n",
    "\n",
    "# heatmap plot\n",
    "ax = figure.add_subplot(3,2,4)\n",
    "grid = np.zeros((len(wires), len(phis2)))\n",
    "for i, gamma in enumerate(wires):\n",
    "    for j, phi in enumerate(phis2):\n",
    "        fname = file_template.format(save_dir, condition, phi, gamma)\n",
    "        fp = open(fname, \"rb\")\n",
    "        data = pickle.load(fp)\n",
    "        fp.close()\n",
    "        grid[i, j] = np.mean(data)\n",
    "bot_model.draw_heatmap(ax, grid, xs, ys, xlabel, ylabel, cmap, diversity_pic_title)\n",
    "\n",
    "### 3. kendall ###\n",
    "file_template = \"{}/kendall_{}_phi{}_gamma{}.pkl\"\n",
    "\n",
    "# distr plot\n",
    "ax = figure.add_subplot(3,2,5)\n",
    "for idx, phi in enumerate(phis1):\n",
    "    kendalls = []\n",
    "    stds = []\n",
    "    for gamma in wires:\n",
    "        fname = file_template.format(save_dir, condition, phi, gamma)\n",
    "        fp = open(fname, \"rb\")\n",
    "        data = pickle.load(fp)\n",
    "        fp.close()\n",
    "        kendalls.append(np.mean(data))\n",
    "        stds.append(np.std(data))\n",
    "    ax.plot(new_wires, kendalls, marker=markers[idx], label='$\\\\phi$:'+str(h))\n",
    "\n",
    "ax.set_xlabel('$\\\\gamma$', fontsize=14)\n",
    "ax.set_ylabel('Discriminative power', fontsize=14)\n",
    "ax.set_xscale('log')\n",
    "ax.set_xlim((new_wires[0], new_wires[-1]))\n",
    "ax.set_xlim((new_wires[0], new_wires[-1]))\n",
    "# ax.legend(loc='lower left', fontsize=14)\n",
    "\n",
    "# heatmap plot\n",
    "ax = figure.add_subplot(3,2,6)\n",
    "grid = np.zeros((len(wires), len(phis2)))\n",
    "for i, gamma in enumerate(wires):\n",
    "    for j, phi in enumerate(phis2):\n",
    "        fname = file_template.format(save_dir, condition, phi, gamma)\n",
    "        fp = open(fname, \"rb\")\n",
    "        data = pickle.load(fp)\n",
    "        fp.close()\n",
    "        grid[i, j] = np.mean(data)\n",
    "bot_model.draw_heatmap(ax, grid, xs, ys, xlabel, ylabel, cmap, kendall_pic_title)\n",
    "\n",
    "### 4. save plot ###\n",
    "plt.subplots_adjust(left=0.1, right=0.95, top=0.95, bottom=0.05, wspace=0.3, hspace=0.3)\n",
    "plt.savefig(save_dir + \"/all_distr_heatmap.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-14T16:02:49.966704Z",
     "start_time": "2020-05-14T16:02:46.542128Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# popularity distribution plots\n",
    "\n",
    "file_template = \"{}/tracked_memes_{}_phi{}_gamma{}.pkl\"\n",
    "\n",
    "fig, axs = plt.subplots(2, 3, figsize=(14, 8))\n",
    "for i, phi in enumerate([1, 10]):\n",
    "    for j, gamma in enumerate([0.001, 0.005, 0.01]):\n",
    "        fname = file_template.format(save_dir, condition, phi, gamma)\n",
    "        fp = open(fname, \"rb\")\n",
    "        data = pickle.load(fp)\n",
    "        fp.close()\n",
    "\n",
    "        quality, number_selected = zip(*data)\n",
    "\n",
    "        low_quality_pop = []\n",
    "        high_quality_pop = []\n",
    "        for qua, pop in zip(quality, number_selected):\n",
    "            if qua > 0:\n",
    "                high_quality_pop.append(pop)\n",
    "            else:\n",
    "                low_quality_pop.append(pop)\n",
    "\n",
    "        count = bot_model.get_count(high_quality_pop)\n",
    "        distr, sum_ = bot_model.get_distr(count)\n",
    "        h_mids, h_heights = bot_model.getbins(distr, sum_)\n",
    "\n",
    "        count = bot_model.get_count(low_quality_pop)\n",
    "        distr, sum_ = bot_model.get_distr(count)\n",
    "        l_mids, l_heights = bot_model.getbins(distr, sum_)\n",
    "\n",
    "        h_dict = defaultdict(list)\n",
    "        for hm, hh in zip(h_mids, h_heights):\n",
    "            h_dict[hm].append(hh)\n",
    "        l_dict = defaultdict(list)\n",
    "        for lm, lh in zip(l_mids, l_heights):\n",
    "            l_dict[lm].append(lh)\n",
    "\n",
    "        hs = []\n",
    "        for k, v in h_dict.items():\n",
    "            hs.append([k, np.mean(v)])\n",
    "        h_mids, h_heights = zip(*sorted(hs, key=lambda x:x[0]))\n",
    "        ls = []\n",
    "        for k, v in l_dict.items():\n",
    "            ls.append([k, np.mean(v)])\n",
    "        l_mids, l_heights = zip(*sorted(ls, key=lambda x:x[0]))\n",
    "\n",
    "        ax = axs[i][j]\n",
    "        ax.loglog(h_mids, h_heights, marker='s', label='high quality')\n",
    "        ax.loglog(l_mids, l_heights, marker='^', label='low quality')\n",
    "        ax.set_xlabel('popularity', fontsize=14)\n",
    "        ax.set_ylabel('P(popularity)', fontsize=14)\n",
    "        ax.tick_params(labelsize=14)\n",
    "        ax.annotate('$\\\\gamma={}$\\n$\\\\phi={}$'.format(gamma, phi), xy=(0.05, 0.05), xycoords='axes fraction', fontsize=12)\n",
    "        if i == 0 and j == 0:\n",
    "            ax.legend(loc=\"upper right\", fontsize=15)\n",
    "\n",
    "plt.subplots_adjust(left=0.08, right=0.92, top=0.92, wspace=0.3, hspace=0.3)\n",
    "plt.show()\n",
    "plt.savefig(save_dir + \"meme_quality_random_distr.png\")\n",
    "plt.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-25T15:26:19.203080Z",
     "start_time": "2020-05-25T15:26:18.292723Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# amplification plot(s)\n",
    "\n",
    "file_template = \"{}/bad_memes_selected_time_{}_phi{}_gamma{}.pkl\"\n",
    "\n",
    "for i, phi in enumerate([1]):\n",
    "    plt.figure(figsize=(10, 5))\n",
    "    for j, gamma in enumerate([0.001]): #[0.5]\n",
    "        fname = file_template.format(save_dir, condition, phi, gamma)\n",
    "        fp = open(fname, \"rb\")\n",
    "        data = pickle.load(fp)\n",
    "        fp.close()\n",
    "        \n",
    "        good_selected = []\n",
    "        bad_selected = []\n",
    "        for _, value in data.items():\n",
    "            if value[0] <= 0 or value[1] <= 0:\n",
    "                continue\n",
    "            good_selected.append(value[0])\n",
    "            bad_selected.append(value[1])\n",
    "\n",
    "        count = dict([val for val in zip(bad_selected, good_selected)])\n",
    "        distr_x, distr_y = bot_model.get_distr(count)\n",
    "        mids, heights = bot_model.getbins(distr_x, distr_y)\n",
    "        ratios = [np.log(height_)/np.log(mid_) for height_, mid_ in zip(heights, mids)]\n",
    "\n",
    "        plt.subplot(121)\n",
    "        plt.loglog(mids, heights, marker='o', label='$\\\\gamma$:'+str(gamma))\n",
    "        plt.subplot(122)\n",
    "        plt.plot(mids, ratios, marker='o', label='$\\\\gamma$:'+str(gamma))\n",
    "        plt.xscale('log')\n",
    "    \n",
    "    # save fig\n",
    "    plt.subplot(121)\n",
    "    plt.loglog([min(mids), max(mids)], [min(mids), max(mids)], '--')\n",
    "    plt.xlabel(\"Bot posts per meme\", fontsize=14)\n",
    "    plt.ylabel(\"Human posts per meme\", fontsize=14)\n",
    "    plt.xticks(fontsize=14)\n",
    "    plt.yticks(fontsize=14)\n",
    "    plt.margins(0.1)\n",
    "    plt.legend(loc='best', fontsize=14)\n",
    "\n",
    "    plt.subplot(122)\n",
    "    plt.xlabel(\"Bot posts per meme\", fontsize=14)\n",
    "    plt.ylabel(\"Exponent $\\\\eta$\", fontsize=14)\n",
    "    plt.xticks(fontsize=14)\n",
    "    plt.yticks(fontsize=14)\n",
    "    plt.margins(0.1)\n",
    "    plt.legend(loc='best', fontsize=14)\n",
    "\n",
    "    plt.subplots_adjust(left=0.1, bottom=0.14, wspace=0.4)\n",
    "    plt.show()\n",
    "    plt.savefig(save_dir + \"bad_meme_selected_random_distr_{}\".format(phi))\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "bot_model_xiaodan.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}