idea2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# load mnist subset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from data import mnist\n",
    "classes, H, e = [0, 1], [784, 16, 1], 1.0\n",
    "train, test = mnist.load(\"train.mat\", classes), mnist.load(\"test.mat\", classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "sam = random.sample(range(len(train[\"x\"])), 500)\n",
    "train[\"x\"] = [train[\"x\"][s] for s in sam]\n",
    "train[\"y\"] = [train[\"y\"][s] for s in sam]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# generations (T), genotypes (G), genes (K)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random, math, numpy as np\n",
    "T, G, K = range(3), range(3), range(3)  # generations (T), genotypes (G), genes (K)\n",
    "generations, genotypes, BETAS = {}, {}, {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "for g in G:        \n",
    "    genes, n = np.zeros(len(K)), int(math.sqrt(len(K)))\n",
    "    genes[random.sample(K, n)] = 1\n",
    "    genotypes[g] = genes\n",
    "    \n",
    "    L, betas = range(len(H)-1), {}\n",
    "    for l in L:\n",
    "        d, c, beta = H[l], H[l+1], {}\n",
    "        for j in range(c):\n",
    "            for i in range(d):\n",
    "                ij = str(i) + \", \" + str(j)\n",
    "                beta[ij] = np.asarray([random.uniform(-1, 1) for _ in K])\n",
    "        betas[l] = beta\n",
    "\n",
    "    BETAS[g] = betas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# create network for idea 2 i.e. compute all the weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# $$W_{ijl}=\\sum_{g=1}^G\\Big(\\beta_{ijlg}\\ \\ gene_{g}\\Big)$$ "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### for all layers $l=1,\\cdots,L$ with $(i,j)=(1,1),\\cdots,(d_l,c_l)$ where $d_l$ and $c_l$ are the numbers of neurons in the previous and the current layers respectively  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def weights(betas, genes, H):\n",
    "    W, w = {}, {}\n",
    "    \n",
    "    L = range(len(H)-1)\n",
    "    for l in L:\n",
    "        beta = betas[l]\n",
    "        d, c = H[l], H[l+1]\n",
    "        for j in range(c):\n",
    "            for i in range(d):\n",
    "                ij = str(i) + \", \" + str(j)\n",
    "                B = beta[ij]\n",
    "                w[ij] = sum(B*genes)\n",
    "        W[l] = w\n",
    "        \n",
    "    return W"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## run all mnist training examples with the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "def ReLU(x):\n",
    "    return np.maximum(0, x)\n",
    "\n",
    "def run(w, H, train):\n",
    "    X, Y = train[\"x\"], train[\"y\"]\n",
    "    Z, L, E = {}, range(len(H)-1), []\n",
    "\n",
    "    for x, y in zip(X, Y):\n",
    "        Z[0] = x\n",
    "        for l in L:\n",
    "            d, c, Z[l+1] = H[l], H[l+1], []\n",
    "            for j in range(c):\n",
    "                w = []\n",
    "                for i in range(d):\n",
    "                    ij = str(i) + \", \" + str(j)\n",
    "                    w.append(W[l][ij])\n",
    "                z = sum(np.asarray(w)*Z[l])\n",
    "                Z[l+1].append(ReLU(z))\n",
    "            Z[l+1] = np.asarray(Z[l+1])\n",
    "        z = np.asarray([sigmoid(v) for v in list(Z[len(L)])])\n",
    "        E.append(0.25-(y-z)*(y-z))\n",
    "    \n",
    "    return float(sum(E)/len(E))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# initial probabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "p, q = {}, {}\n",
    "for k in K:\n",
    "    p[k] = float(1/math.sqrt(len(G)))\n",
    "    q[k] = 1 - p[k]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# idea 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3/3 [01:07<00:00, 22.62s/it]\n",
      "100%|██████████| 3/3 [00:50<00:00, 16.85s/it]\n",
      "100%|██████████| 3/3 [00:49<00:00, 16.39s/it]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "for t in T:\n",
    "    F, f = {}, {}\n",
    "    for g in tqdm(G):\n",
    "        genes = genotypes[g]\n",
    "        betas = BETAS[g]\n",
    "        for k, gene in enumerate(genes):\n",
    "            if gene == 1:\n",
    "                if k not in F.keys():\n",
    "                    F[k] = []\n",
    "                W = weights(betas, genes, H)\n",
    "                F[k].append(run(W, H, train))\n",
    "            else:\n",
    "                if k not in f.keys():\n",
    "                    f[k] = []\n",
    "                W = weights(betas, genes, H)\n",
    "                f[k].append(run(W, H, train))\n",
    "    \n",
    "    for k in K:\n",
    "        if k in F.keys():\n",
    "            F[k] = sum(F[k])/len(F[k])\n",
    "        else:\n",
    "            F[k] = 0\n",
    "        if k in f.keys():\n",
    "            f[k] = sum(f[k])/len(f[k])\n",
    "        else:\n",
    "            f[k] = 0\n",
    "        n = p[k]*(1+e*F[k]) + q[k]*(1+e*f[k])\n",
    "        p[k] = p[k]*(1+e*F[k]) / n\n",
    "    \n",
    "    for g in G:\n",
    "        genes = np.zeros(len(K))\n",
    "        for k in K:\n",
    "            r = random.uniform(0,1)\n",
    "            if r <= p[k]:\n",
    "                genes[k] = 1\n",
    "        genotypes[g] = genes      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# evaluate on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.22475020641992638\n",
      "-0.03310871826431865\n",
      "0.06817998454775216\n"
     ]
    }
   ],
   "source": [
    "for g in G:\n",
    "    genes = genotypes[g]\n",
    "    betas = BETAS[g]\n",
    "    W = weights(betas, genes, H)\n",
    "    print(run(W, H, test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ['r': run all cells](https://stackoverflow.com/questions/33143753/jupyter-ipython-notebooks-shortcut-for-run-all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "Jupyter.keyboard_manager.command_shortcuts.add_shortcut('r', {\n",
       "    help : 'run all cells',\n",
       "    help_index : 'zz',\n",
       "    handler : function (event) {\n",
       "        IPython.notebook.execute_all_cells();\n",
       "        return false;\n",
       "    }}\n",
       ");"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%javascript\n",
    "\n",
    "Jupyter.keyboard_manager.command_shortcuts.add_shortcut('r', {\n",
    "    help : 'run all cells',\n",
    "    help_index : 'zz',\n",
    "    handler : function (event) {\n",
    "        IPython.notebook.execute_all_cells();\n",
    "        return false;\n",
    "    }}\n",
    ");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}