the_basics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Basic Reinforcement Learning\n",
    "Purpose: To learn about RL from its most basic instantiation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import time\n",
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('LunarLander-v2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('CartPole-v0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<gym.core.Env at 0x7f9a895c1af0>"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gym.Env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.03158174,  0.01500655,  0.04988515, -0.03702167])"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.reset()\n",
    "env.state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Vector_env(gym.Wrapper):\n",
    "    '''\n",
    "     class Vector_env\n",
    "     \n",
    "     Acts as a wrapper for the OpenAI gym environments.\n",
    "    '''\n",
    "    def __init__(self, env, n=1):\n",
    "        '''\n",
    "         venv = Vector_env(env, n=1)\n",
    "         \n",
    "         Creates n copies of env.\n",
    "        '''\n",
    "        super().__init__(env)\n",
    "        self.envs = []\n",
    "        for k in range(n):\n",
    "            env_r = copy.deepcopy(env)\n",
    "            env_r.seed(np.random.randint(10000))\n",
    "            self.envs.append(env_r)\n",
    "    \n",
    "    def reset(self):\n",
    "        '''\n",
    "         states = venv.reset()\n",
    "         \n",
    "         Resets all the environments, and returns an array with the\n",
    "         reset states in each row.\n",
    "        '''\n",
    "        states = []\n",
    "        for e in self.envs:\n",
    "            state = e.reset()\n",
    "            states.append(state)\n",
    "        return np.array(states)\n",
    "    \n",
    "    def step(self, actions):\n",
    "        '''\n",
    "         S, R, done, info = venv.step(actions)\n",
    "         \n",
    "         Takes one step in each of the environments, and returns an array\n",
    "         for each of the resulting outputs.\n",
    "         \n",
    "         Inputs:\n",
    "           actions   an array of actions\n",
    "           \n",
    "         Outputs:\n",
    "           S    array of states after the step\n",
    "           R    array of rewards from the step\n",
    "           done array of Boolean flags:\n",
    "                  False means episode continues\n",
    "                  True means episode is done\n",
    "        '''\n",
    "        S, R, done = [], [], []\n",
    "        for e,a in zip(self.envs, actions):\n",
    "            s, r, d, inf = e.step(a)\n",
    "            S.append(s)\n",
    "            R.append(r)\n",
    "            done.append(d)\n",
    "        return np.array(S), np.array(R), np.array(done)\n",
    "    \n",
    "    def state(self):\n",
    "        S = []\n",
    "        for e in self.envs:\n",
    "            S.append(e.state)\n",
    "        return np.array(S)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent():\n",
    "    '''\n",
    "     class Agent\n",
    "     \n",
    "     Represents an RL agent. Its Q function is represented by a neural network, so that\n",
    "       agent.Q(s) returns a vector of probabilities over action-space.\n",
    "    '''\n",
    "    def __init__(self, envs):\n",
    "        '''\n",
    "         ag = Agent(envs)\n",
    "         \n",
    "         Instantiates an Agent object.\n",
    "         \n",
    "         Inputs:\n",
    "           envs  a Vector_env object, containing the environments for the task\n",
    "        '''\n",
    "        self.envs = envs\n",
    "        self.state_dim = self.envs.observation_space.shape[0]  #state_dim\n",
    "        self.n_actions = self.envs.action_space.n  #n_actions\n",
    "        #self.state = torch.zeros(self.state_dim)\n",
    "        self._max_episode_steps = 100\n",
    "        \n",
    "        # Policy\n",
    "        self.hidden = 100\n",
    "        self.Pi_net = torch.nn.Sequential(\n",
    "                    torch.nn.Linear(self.state_dim, self.hidden),\n",
    "                    torch.nn.Sigmoid(),\n",
    "                    torch.nn.Linear(self.hidden, self.n_actions))\n",
    "        \n",
    "        # Value function\n",
    "        self.V_hidden = 80\n",
    "        self.V_net = torch.nn.Sequential(\n",
    "                    torch.nn.Linear(self.state_dim, self.V_hidden),\n",
    "                    torch.nn.Sigmoid(),\n",
    "                    torch.nn.Linear(self.V_hidden, 1))\n",
    "        \n",
    "        # Eligibility-trace buffer for Pi_net\n",
    "        self.Piz = []\n",
    "        for p in self.Pi_net.parameters():\n",
    "            self.Piz.append(torch.zeros_like(p))\n",
    "            \n",
    "        # Eligibility-trace buffer for V_net\n",
    "        self.Vz = []\n",
    "        for p in self.V_net.parameters():\n",
    "            self.Vz.append(torch.zeros_like(p))\n",
    "        \n",
    "    def Pi(self, states=None):\n",
    "        if states is None:\n",
    "            S = torch.tensor(self.envs.state(), dtype=torch.float)\n",
    "        else:\n",
    "            S = torch.tensor(states, dtype=torch.float)\n",
    "        return torch.distributions.Categorical(logits=self.Pi_net(S))\n",
    "    \n",
    "    def V(self, states=None):\n",
    "        if states is None:\n",
    "            S = torch.from_numpy(self.envs.state())\n",
    "        else:\n",
    "            S = torch.tensor(states, dtype=torch.float)\n",
    "        return self.V_net(S)\n",
    "    \n",
    "    \n",
    "    def ChooseAction(self, state=None, eps=0.):\n",
    "        '''\n",
    "         A = ag.ChooseAction(state, eps=0.)\n",
    "         \n",
    "         Choose an action, given the state.\n",
    "         With probability eps, it will choose an action randomly.\n",
    "         Otherwise, it will choose an action that yields the highest Q-value.\n",
    "        '''\n",
    "        if state is None:\n",
    "            state = self.envs.state()\n",
    "            \n",
    "        batchsize = len(state)\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            Pa = self.Pi(state)\n",
    "        A = Pa.sample().detach().numpy()\n",
    "        \n",
    "        # With prob eps, choose random action\n",
    "        A_random = np.array(np.random.randint(0, high=self.n_actions, size=batchsize), dtype=np.float)\n",
    "        r = np.random.rand(batchsize)\n",
    "        ridx = r<eps\n",
    "        A[ridx] = A_random[ridx]\n",
    "        \n",
    "        return A\n",
    "    \n",
    "    def Episode(self, gamma=1., eps=0.):\n",
    "        '''\n",
    "         traj = ag.Episode(gamma=1.)\n",
    "         \n",
    "         Runs one episode, and returns the trajectory. The environment is\n",
    "         automatically reset at the beginning of the episode.\n",
    "                 \n",
    "         Inputs:\n",
    "           gamma  reward discount\n",
    "         \n",
    "         Outputs:\n",
    "           traj trajectory, as a list of lists of the form\n",
    "                [A, S, Snext, R, G, done, info]\n",
    "                A     action taken at state S\n",
    "                S     state\n",
    "                Snext state after taking action A\n",
    "                R     reward\n",
    "                G     future return (discounted)\n",
    "                done  flag to indicate episode is done\n",
    "                info  not sure ?\n",
    "        '''\n",
    "        S = self.envs.reset()\n",
    "        traj = []\n",
    "        bs = len(S)  # batchsize\n",
    "        for k in range(self._max_episode_steps):\n",
    "            action = self.ChooseAction(state=S, eps=eps)\n",
    "            Snext, r, done = self.envs.step(action)\n",
    "            traj.append(dict(action=action, S=S, Snext=Snext, r=r, G=np.zeros(bs), done=done))\n",
    "            S = Snext\n",
    "            if done.all():\n",
    "                break\n",
    "        # Use that trajectory (and rewards) to update the policy parameters\n",
    "        # Process the return for all visited states\n",
    "        #traj[-1][4] = np.zeros(bs)\n",
    "        for k in range(len(traj)-2, -1, -1):\n",
    "            traj[k]['G'] = traj[k]['r'] + (1.-traj[k]['done'])*gamma*traj[k+1]['G']\n",
    "        return traj\n",
    "    \n",
    "    \n",
    "\n",
    "    def Simulate(self, trials=1, eps=0.):\n",
    "        time.sleep(1)\n",
    "        avg_k = 0.\n",
    "        for trial in range(trials):\n",
    "            observation = self.envs.envs[0].reset()\n",
    "            for k in range(self._max_episode_steps):\n",
    "                self.envs.envs[0].render()\n",
    "                action = self.ChooseAction(state=[observation], eps=eps)\n",
    "\n",
    "                observation, reward, done, info = self.envs.envs[0].step(action[0])\n",
    "\n",
    "                if done:\n",
    "                    print('Done after '+str(k))\n",
    "                    avg_k += k\n",
    "                    break\n",
    "        print('Average reward: '+str(avg_k/trials))\n",
    "        time.sleep(1)\n",
    "        self.envs.envs[0].close()\n",
    "        \n",
    "        \n",
    "    def REINFORCE(self, gamma=1.0, alpha=0.0001, eps=0.):\n",
    "        '''\n",
    "         ag.REINFORCE(env, gamma=1.0, alpha=0.0001, eps=0.)\n",
    "         \n",
    "         Performs an episode of the REINFORCE algorithm, following a reset\n",
    "         of the environment.\n",
    "         \n",
    "         Inputs:\n",
    "           gamma  discount rate for rewards (default 1.0)\n",
    "           alpha  step size for gradient ascent (the policy gradient)\n",
    "           eps    probability of choosing a random action\n",
    "           \n",
    "         Output:\n",
    "           traj   trajectory, a list of lists of the form:\n",
    "                [A, S, Snext, R, G, done, info]\n",
    "                A     action taken at state S\n",
    "                S     state\n",
    "                Snext state after taking action A\n",
    "                R     reward\n",
    "                G     future return (discounted)\n",
    "                done  flag to indicate episode is done\n",
    "                info  not sure ?\n",
    "        '''\n",
    "        # Perform an episode and record the trajectory\n",
    "        traj = self.Episode(eps=eps)\n",
    "        bs = len(traj[0]['action'])\n",
    "                \n",
    "        # Update policy parameters\n",
    "        loss = 0.\n",
    "        for k,blah in enumerate(traj):\n",
    "            A = blah['action']  # actions\n",
    "            G = torch.tensor(blah['G'], dtype=torch.float)  # future return\n",
    "            S = blah['S']  # before states\n",
    "            dones = blah['done']\n",
    "            Pa = self.Pi(S)  # get action probabilities\n",
    "            logPa_x_G = Pa.log_prob(torch.tensor(A)) * (G)\n",
    "            loss += torch.mean(logPa_x_G)\n",
    "            #print(loss)\n",
    "        \n",
    "        self.Pi_net.zero_grad()\n",
    "        loss.backward()\n",
    "            \n",
    "        with torch.no_grad():\n",
    "            for p in self.Pi_net.parameters():\n",
    "                p += alpha * p.grad\n",
    "        return traj\n",
    "\n",
    "    \n",
    "    def REINFORCE_baseline(self, gamma=1.0, alpha=0.0001, alpha_v=0.0001, eps=0.1):\n",
    "        '''\n",
    "         ag.REINFORCE_baseline(env, gamma=1.0, alpha=0.0001, alpha_v=0.0001)\n",
    "         \n",
    "         Performs an episode of the REINFORCE algorithm with baseline, following\n",
    "         a reset of the environment. This method computes a model of the value\n",
    "         function, in addition to a policy.\n",
    "         \n",
    "         Inputs:\n",
    "           gamma  discount rate for rewards (default 1.0)\n",
    "           alpha  step size for gradient ascent (the policy gradient)\n",
    "           alpha_v step size for gradient ascent of the value function\n",
    "           \n",
    "         Output:\n",
    "           traj   trajectory, a list of lists of the form:\n",
    "                  [A, S, Snext, R, G, done, info]\n",
    "                  A     action taken at state S\n",
    "                  S     state\n",
    "                  Snext state after taking action A\n",
    "                  R     reward\n",
    "                  G     future return (discounted)\n",
    "                  done  flag to indicate episode is done\n",
    "                  info  not sure ?\n",
    "        '''\n",
    "        v_mult = 1.\n",
    "        \n",
    "        # Perform an episode and record the trajectory\n",
    "        traj = self.Episode(eps=eps(0))\n",
    "        bs = len(traj[0]['action'])\n",
    "                \n",
    "        # Update policy parameters\n",
    "        gamma_t = gamma\n",
    "        Pa_loss = 0.\n",
    "        V_loss = 0.\n",
    "        for k,blah in enumerate(traj):\n",
    "            A = blah['action']\n",
    "            S = blah['S']\n",
    "            G = torch.from_numpy(blah['G'])  # future return\n",
    "            G.requires_grad = False\n",
    "            dones = blah['done']\n",
    "            \n",
    "            # Policy network\n",
    "            Pa = self.Pi(S)  # returns a distribution object\n",
    "            \n",
    "            # Value network\n",
    "            V = self.V(S)*v_mult\n",
    "            \n",
    "            # Error\n",
    "            with torch.no_grad():\n",
    "                err = G - V\n",
    "            \n",
    "            # Recall that d(log(Pi)) = dPi / Pi\n",
    "            logPa_x_err = Pa.log_prob(torch.from_numpy(A)) * err * gamma_t\n",
    "            \n",
    "            Pa_loss += torch.mean(logPa_x_err)\n",
    "            V_loss += torch.mean( (G-V)**2 ) * gamma_t\n",
    "            \n",
    "            gamma_t *= gamma\n",
    "        \n",
    "        # Backproject gradient\n",
    "        self.Pi_net.zero_grad()\n",
    "        self.V_net.zero_grad()\n",
    "        Pa_loss.backward()\n",
    "        V_loss.backward()\n",
    "        with torch.no_grad():\n",
    "            for p in self.Pi_net.parameters():\n",
    "                p += alpha * p.grad\n",
    "            for p in self.V_net.parameters():\n",
    "                p -= alpha_v * p.grad\n",
    "            \n",
    "        return traj\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": "false"
   },
   "source": [
    "### Cart-Pole"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "#env = gym.make('LunarLander-v2')\n",
    "S = env.reset()\n",
    "envs = Vector_env(env, n=32)\n",
    "ag_r = Agent(envs)\n",
    "ag_r._max_episode_steps = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = []\n",
    "counter = 0\n",
    "mysum = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "interval = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "450\n",
      "500\n",
      "550\n",
      "600\n",
      "650\n",
      "700\n",
      "750\n",
      "800\n"
     ]
    }
   ],
   "source": [
    "for i_episode in range(400):\n",
    "    traj = ag_r.REINFORCE_baseline(alpha=0.0001, alpha_v=0.0001, eps=(lambda x: 0.1))\n",
    "    #traj = ag_r.REINFORCE(alpha=0.0001, eps=0.)\n",
    "    counter += 1\n",
    "    mysum += np.mean(traj[0]['G'])\n",
    "    if counter%interval==0:\n",
    "        print(counter)\n",
    "        score.append(mysum/interval)\n",
    "        mysum = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD6CAYAAABamQdMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXTU9b3/8eebEMjCEkIW9lXC4gJoRNS64l7rUq9Wal2qLfWqrd5qf2731p7beo/X1rba7V57a8VbAXGrnrbaArVqryAERLbIIiYkEJKQAAmE7O/fHxlogEBCZsJ3ZvJ6nJMzM5+ZSV4hmReffOa7mLsjIiLxpUfQAUREJPJU7iIicUjlLiISh1TuIiJxSOUuIhKHVO4iInGo3XI3s+Fm9o6Z5ZvZWjO7NzSebmYLzGxj6HJAq+c8bGabzGy9mV3ald+AiIgcztrbzt3MBgOD3X2FmfUFlgPXALcBle7+hJk9BAxw9wfNbBIwF5gGDAEWAjnu3nSkr5GRkeGjRo2KxPcjItJtLF++fIe7Z7Z1X8/2nuzuJUBJ6Hq1meUDQ4GrgfNDD5sN/A14MDQ+z93rgM/MbBMtRb/4SF9j1KhR5OXldfT7ERERwMwKj3TfMa25m9koYCrwIZAdKv79/wFkhR42FChq9bTi0JiIiBwnHS53M+sDvArc5+5VR3toG2OHrf2Y2SwzyzOzvPLy8o7GEBGRDuhQuZtZIi3F/qK7vxYaLg2tx+9fly8LjRcDw1s9fRiw7dDP6e7Punuuu+dmZra5ZCQiIp3Uka1lDPgNkO/uP25115vAraHrtwJvtBq/0cx6m9loYBywNHKRRUSkPe2+oQqcDdwMrDazlaGxR4AngPlmdgewBbgewN3Xmtl8YB3QCNx9tC1lREQk8jqytczfaXsdHWDGEZ7zOPB4GLlERCQM2kNVRCQOdWRZRkREIqi52VlfWs2SzRVk9U3i86cMjvjXULmLiHSx5mYnf3sVH26uZMnmCpYWVLKrpgGAqyYPUbmLiMSCpmYnv6SKJZsr+PCzSpZ+VsnufS1lPnJgCpdMymb6mIGcMWYgQ9OSuySDyl1EJEyty3zJ5gqWflZJVW0jAKMGpnD5SYM4Y0w6Z4weyJAuKvNDqdxFRI5RY1Mz6/bPzDdXsrSgkupQmY/OSOXzpwxumZmPHsig/kmBZFS5i4gcQW1DE2VVdZTs3sf2qlqKd+5jeeFOln1WSXVdS5mPyUzlylOGMH1MOtPHDCS7XzBlfiiVu4h0O+5O1b5GtlfVUrJ7H6VVtWzfXcf2qn1s313L9qo6tu/ex87Qm56tjc1M5aopQ0Iz83SyoqTMD6VyF5G4VF5dx/LCnWzfve9AWW+vqqU0NBOvbWg+7DkZfXqR3S+JoWlJnDoijUH9khjUP/QRut43KTGA7+bYqdxFJC64O5vK9rAgv5SF60r5qGgX+89FlJhgZPdrKegTh/RjxoSsg0o7O/TRq2f87NepcheRmNXY1Myygp0szC9lYX4phRU1AJw8tD/3zcjh3JwMhqenkJ7Six49jnQUlfikcheRmFJV28B7G8pZuK6Ud9aXs3tfA7169uDssQOZde4YZkzIDmwLlWiicheRqFe8s4ZF+WUszC9lyeYKGpqc9NReXDwpm4smZnPOuAxSe6vOWtO/hohEneZmZ8223SxcV8qC/DLyS1pO/jY2M5XbPzeaiydmM3XEABK62VLLsVC5i0hUqGts4oNNFSzIL2VRfimlVXX0MMgdlc6jV0xkxsQsxmT2CTpmzFC5i0jgtu7ax1d/u5QNpXtI7ZXAeeMzuWhiNheMz2JAaq+g48UklbuIBGrttt189bfL2NfQxC9vOpUZE7Po3TMh6FgxT+UuIoF5d0M5d/1uOf2TE3n1n88iJ7tv0JHihspdRAIxf1kRD7++mpzsvjz/1dOj5pgs8ULlLiLHlbvz04UbeXrRRs4Zl8Evbzo1ZnbpjyXtlruZPQdcCZS5+0mhsZeA8aGHpAG73H2KmY0C8oH1ofuWuPudkQ4tIrGpoamZR15bzcvLi7n+tGH8xxdPJjEhfnb5jyYdmbk/D/wceGH/gLt/af91M3sK2N3q8Z+6+5RIBRSR+FBd28BdL67g/Y07uO+icdw7Yxxm2k69q7Rb7u7+XmhGfhhr+cncAFwY2VgiEk9Kq2q57bfL2FBazZPXncINpw8POlLcC/fvoXOAUnff2GpstJl9ZGbvmtk5R3qimc0yszwzyysvLw8zhohEqw2l1Vz7i/9jS8VenrvtdBX7cRJuuc8E5ra6XQKMcPepwLeBOWbWr60nuvuz7p7r7rmZmZlhxhCRaPTBpzu47lcf0NjszL/zTM7L0Wv9eOn01jJm1hP4InDa/jF3rwPqQteXm9mnQA6QF2ZOEYkxb6zcygMvf8yogak8f/s0hh6nE0NLi3A2hbwI+MTdi/cPmFkmUOnuTWY2BhgHbA4zo4jEEHfnV+9+ypNvr+eM0ek8e3Mu/VO0qePx1u6yjJnNBRYD482s2MzuCN11IwcvyQCcC6wys4+BV4A73b0ykoFFJHo1NjXzb2+s4cm313PV5CG8cMc0FXtAOrK1zMwjjN/WxtirwKvhxxKRWFNT38g353zEok/K+Ofzx/KdS8Z3u7MfRRPtoSoiYSuvruNrs5exeutuvn/NSdw8fWTQkbo9lbuIhGVz+R5u/e1Syqvr+O+bc7l4UnbQkQSVu4iEYXlhJV+bnUcPM+bNOpMpw9OCjiQhKncR6ZS315Rw77yVDElL5vmvns7IgalBR5JWVO4icsyWFVRy95yPmDysP/9z6+mk62xJUUflLiLHpHJvPd+c8xHDBiQz+/ZpOlxvlFK5i0iHNTc7989fSeXeel676ywVexTTgZRFpMN+/f5m3llfzqOfn8hJQ/sHHUeOQuUuIh2yvHAnT/55PZedOIhbztR27NFO5S4i7dpVU8+35n7EkLQk/vOfTtFJNmKA1txF5KjcnQdeXkVZdS2v3HkW/ZO1zh4LNHMXkaN67v8KWJhfykOXT2SydlKKGSp3ETmij4t28cRb+Vw8KZvbzx4VdBw5Bip3EWnT7n0N3D1nBVl9k/ih1tljjtbcReQw7s6Dr6xi++5a5t95Jmkp2gM11mjmLiKHeWFxIW+v3c7/u2w8p44YEHQc6QSVu4gcZM3W3Tz+x3wuGJ/J1z43Jug40kkqdxE5oLq2ZZ09PbUXT90wRWdSimFacxcRoGWd/aHXVlO8cx/zZk3XkR5jnGbuIgLAix9u4Y+rSvj2xTmcPio96DgSpnbL3cyeM7MyM1vTaux7ZrbVzFaGPq5odd/DZrbJzNab2aVdFVxEImfdtir+/Q/rODcnk38+b2zQcSQCOjJzfx64rI3xn7j7lNDHnwDMbBJwI3Bi6Dm/NLOESIUVkcjbU9fIPXNWkJacyI9vmKx19jjRbrm7+3tAZQc/39XAPHevc/fPgE3AtDDyiUgXcnf+9fXVFFTs5ZmZU8no0zvoSBIh4ay532Nmq0LLNvs3hB0KFLV6THFo7DBmNsvM8swsr7y8PIwYItJZ8/OK+P3Kbdx3UQ7TxwwMOo5EUGfL/VfAWGAKUAI8FRpv6+85b+sTuPuz7p7r7rmZmZmdjCEinbV+ezWPvbmWs08YyN0XnBB0HImwTpW7u5e6e5O7NwO/5h9LL8XA8FYPHQZsCy+iiERaTX0jd89ZQZ/eifz0S1NJ0Dp73OlUuZvZ4FY3rwX2b0nzJnCjmfU2s9HAOGBpeBFFJNL+7fdr+bR8D0/fOIXMvlpnj0ft7sRkZnOB84EMMysGHgPON7MptCy5FADfAHD3tWY2H1gHNAJ3u3tT10QXkc54ZXkxr64o5lszxnH2CRlBx5Eu0m65u/vMNoZ/c5THPw48Hk4oEekaG0ur+bffr+GM0encO2Nc0HGkC2kPVZFuYl99E3fPWUFKrwSemal19ninY8uIdBPfe3MtG0r3MPv2aWT3Swo6jnQxzdxFuoH3NpTzUl4Rd50/lvNytOlxd6ByF+kGfvt/n5HZtzf3XZQTdBQ5TlTuInGusGIvf9tQzsxpI+jVUy/57kI/aZE497slhSSYcdMZI4KOIseRyl0kju2rb2J+XjGXnjhIb6J2Myp3kTj25sdb2b2vgVvOHBl0FDnOVO4iccrdeWFxIeOz+zJttM6s1N2o3EXi1Iotu1i7rYpbzhqJmXZY6m5U7iJx6oXFBfTt3ZNrprR5SgWJcyp3kThUXl3Hn1aXcN1pw0jtrR3RuyOVu0gcmrd0Cw1Nzs16I7XbUrmLxJnGpmZe/HAL54zLYGxmn6DjSEBU7iJxZsG6UrZX1XLLmaOCjiIBUrmLxJkXFhcyNC2ZCydkBR1FAqRyF4kjG0urWby5gq9MH6njtXdzKneROPLC4kJ69ezBl04f3v6DJa6p3EXiRHVtA6+tKObKUwaTntor6DgSMJW7SJx4bcVW9tY3caveSBU6UO5m9pyZlZnZmlZjPzSzT8xslZm9bmZpofFRZrbPzFaGPv6rK8OLSAt353+XFDJ5WH8mD08LOo5EgY7M3J8HLjtkbAFwkrufAmwAHm5136fuPiX0cWdkYorI0Sz+tIJNZXu0+aMc0G65u/t7QOUhY39x98bQzSXAsC7IJiIdNHtxAempvfj8KYODjiJRIhJr7rcDb7W6PdrMPjKzd83snCM9ycxmmVmemeWVl5dHIIZI97R11z4WrCvlS6cPJykxIeg4EiXCKnczexRoBF4MDZUAI9x9KvBtYI6Z9Wvrue7+rLvnuntuZqbOxi7SWXM+LATQafTkIJ0udzO7FbgSuMndHcDd69y9InR9OfApoNOti3SRusYm5i0tYsbEbIYNSAk6jkSRTpW7mV0GPAhc5e41rcYzzSwhdH0MMA7YHImgInK4P60uoWJvvU6jJ4dp90DPZjYXOB/IMLNi4DFato7pDSwIneFlSWjLmHOBfzezRqAJuNPdK9v8xCISthcWFzImI5Wzx2YEHUWiTLvl7u4z2xj+zREe+yrwarihRKR9q4t389GWXTz2hUn00HFk5BDaQ1UkRr2wuICUXglcd5q2RJbDqdxFYtDOvfW8+fE2rp06lH5JiUHHkSikcheJQfPziqhrbNYeqXJEKneRGNPU7Pzuw0LOGJ3O+EF9g44jUUrlLhJj3t1QRlHlPs3a5ahU7iIxZvYHhWT3680lJ2YHHUWimMpdJIYU7NjLuxvK+fK0kSQm6OUrR6bfDpEY8r9LCunZw5g5TafRk6NTuYvEiJr6Rl7OK+LykweT1S8p6DgS5VTuIjHijZXbqKpt1HFkpENU7iIxwN15YXEhEwf3I3fkgKDjSAxQuYvEgOWFO8kvqeKWM0cSOlifyFGp3EViwOzFhfRN6snVU4YEHUVihMpdJMqVVdXy1uoSbsgdTkqvdg/kKgKo3EWi3tylRTQ2O1+ZrjdSpeNU7iJRrKGpmTlLCzkvJ5PRGalBx5EYonIXiWIL1pVSWlWnzR/lmKncRaLY7A8KGJ6ezPnjs4KOIjFG5S4SpdZvr+bDzyr5yhkjSdBp9OQYtVvuZvacmZWZ2ZpWY+lmtsDMNoYuB7S672Ez22Rm683s0q4KLhLvXlhcQO+ePbghV8eRkWPXkZn788Blh4w9BCxy93HAotBtzGwScCNwYug5vzSzhIilFekmqmobeP2jrVw1eQgDUnsFHUdiULvl7u7vAZWHDF8NzA5dnw1c02p8nrvXuftnwCZgWoSyinQbLy0toqa+SSfkkE7r7Jp7truXAIQu97/bMxQoavW44tCYiHTQJ9ureGrBes4Zl8HJw/oHHUdiVKTfUG3rXR9v84Fms8wsz8zyysvLIxxDJDbtqWvkrhdX0DcpkadumBx0HIlhnS33UjMbDBC6LAuNFwOt3/0ZBmxr6xO4+7PunuvuuZmZmZ2MIRI/3J2HXl1FwY69/GzmVLL66pjt0nmdLfc3gVtD128F3mg1fqOZ9Taz0cA4YGl4EUW6h98tKeQPq0p44NLxTB8zMOg4EuPaPQqRmc0FzgcyzKwYeAx4AphvZncAW4DrAdx9rZnNB9YBjcDd7t7URdlF4saq4l18/w/5XDA+kzvPHRt0HIkD7Za7u888wl0zjvD4x4HHwwkl0p3srmngrhdXkNm3Nz++YQo9tMOSRICOHyoSoOZm5/6XV1JaVcv8b5ypbdolYnT4AZEA/fr9zSzML+ORKyYydYROnyeRo3IXCcjSzyp58s/r+fzJg7ntrFFBx5E4o3IXCcCOPXV8c+4Khg9I5onrTtZ5USXiVO4ix1lTs3PfvJXsqmnglzedRt+kxKAjSRzSG6oix9kzizby9007+M/rTmbSkH5Bx5E4pZm7yHH03oZynvnrRq47dZgO5StdSuUucpxs313LfS+tJCerLz+45iSts0uXUrmLHAcNTc3cM2cFdQ1N/OKmU0nupdMcSNfSmrvIcfDDP68nr3Anz8ycyglZfYKOI92AZu4iXewva7fz7HubuXn6SK6aPCToONJNqNxFulBRZQ0PvPwxJw/tz79eOTHoONKNqNxFukhtQxN3vbgCgF/edCq9e2qdXY4frbmLdJEf/HEdq7fu5te35DI8PSXoONLNaOYu0gXeWLmV3y3Zwqxzx3DxpOyg40g3pHIXibBNZXt4+LXVnD5qAN+5dHzQcaSbUrmLRFBNfSN3vbicpMQEfjbzVBIT9BKTYGjNXSRC3J1//f0aNpbt4YXbpzGov05wLcHRtEIkQubnFfHaiq1868JxnDMuM+g40s2p3EUiYN22Kr77xlo+d0IG35oxLug4Ip1fljGz8cBLrYbGAN8F0oCvA+Wh8Ufc/U+dTigS5fbVN3HPnBWkpSTy0xunkKATXEsU6HS5u/t6YAqAmSUAW4HXga8CP3H3H0UkoUiU+8+3P2Hzjr3M+foZZPTpHXQcESByyzIzgE/dvTBCn08kJnzw6Q6e/6CA284axVljM4KOI3JApMr9RmBuq9v3mNkqM3vOzHRKd4lL1bUNfOflVYzOSOXByyYEHUfkIGGXu5n1Aq4CXg4N/QoYS8uSTQnw1BGeN8vM8swsr7y8vK2HiES1x/+YT8nuffzo+sk6PrtEnUjM3C8HVrh7KYC7l7p7k7s3A78GprX1JHd/1t1z3T03M1ObjUlseeeTMuYtK2LWuWM5baT+OJXoE4lyn0mrJRkzG9zqvmuBNRH4GiJRY1dNPQ++uoqc7D78y8Xa7FGiU1h7qJpZCnAx8I1Ww0+a2RTAgYJD7hOJed97cy2Ve+t57rbTdRhfiVphlbu71wADDxm7OaxEIlHsrdUl/H7lNv7lohxOGto/6DgiR6Q9VEU6aMeeOh79/RpOHtqfuy4YG3QckaNSuYt0gLvz6Our2VPbyFM3TNbRHiXq6TdUpAPeWLmNP68t5f5LcsjJ7ht0HJF2qdxF2rF9dy3ffWMNp40cwNfOGRN0HJEOUbmLHIW78+Crq2hocp66frIOCiYxQ+UuchTzlhXx7oZyHrp8AqMyUoOOI9JhKneRIyiqrOEHf1jHWWMHcvP0kUHHETkmKneRNjQ3O9955WPMjCf/6RR6aDlGYozKXaQNsxcXsGRzJd+9chLDBqQEHUfkmKncRQ7xafkennjrEy6ckMX1ucOCjiPSKSp3kVaamp0HXv6YpMQEnvjiyZhpOUZiU1jHlhGJN8++t5mPtuzi6RunkNUvKeg4Ip2mmbtIyCfbq/jJgg1ccfIgrpo8JOg4ImFRuYsA9Y3N3D//Y/ol9+T7V5+k5RiJeVqWEQF+/s4m1m6r4r9vPo2BfXoHHUckbJq5S7e3qngXv3hnE1+cOpRLTxwUdByRiFC5S7dW29DE/fM/JrNPbx77wolBxxGJGC3LSLf2kwUb2Fi2h9m3T6N/SmLQcUQiRjN36bbyCip59v3NfPmMEZyXkxl0HJGIUrlLt1RT38j9L3/M0LRkHrliYtBxRCIurGUZMysAqoEmoNHdc80sHXgJGAUUADe4+87wYopE1hNvfUJhRQ3zZk2nT2+tTkr8icTM/QJ3n+LuuaHbDwGL3H0csCh0WyQqrCzaxR3PL+OFxYXcfvZopo8ZGHQkkS7RFVOWq4HzQ9dnA38DHuyCryPSYcsLd/LMoo28u6GctJREHrgkh6+fq1PmSfwKt9wd+IuZOfDf7v4skO3uJQDuXmJmWW090cxmAbMARowYEWYMkbZ9uLmCn/11E3/ftIP01F48eNkEbj5zpJZiJO6F+xt+trtvCxX4AjP7pKNPDP1H8CxAbm6uh5lD5AB3Z/HmCp5euJEPP6sko09vHr1iIjdNH0FKL5W6dA9h/aa7+7bQZZmZvQ5MA0rNbHBo1j4YKItATpF2uTvvb9zBM4s2kle4k+x+vXnsC5OYOW0ESYkJQccTOa46Xe5mlgr0cPfq0PVLgH8H3gRuBZ4IXb4RiaAiR+Lu/G19OU8v2sjKol0M7p/E968+ketzh6vUpdsKZ+aeDbweOnpeT2COu79tZsuA+WZ2B7AFuD78mCKHc3cW5pfxzKKNrN66m6FpyfzHtSdz3WlD6d1TpS7dW6fL3d03A5PbGK8AZoQTSuRompudv6zbztOLNpFfUsWI9BSevO4Urj11KIkJ2i9PBHRsGYkhTc3On1aX8PO/bmJ9aTWjM1J56vrJXD1lCD1V6iIHUblL1GtsauYPq0r42V838mn5Xk7I6sPTN07hylOGkNBDJ9UQaYvKXaJW8c4a5i8r4qW8Ikqr6hif3ZdffPlULj9pED1U6iJHpXKXqNLQ1MxfPylj7tItvLuhHIDzczL5wTUjmTEhS6Uu0kEqd4kKRZU1vLSsiPl5RZRV1zGoXxLfvHAcN+QOY9iAlKDjicQclbsEpqGpmUX5ZcxZuoX3N5ZjwAXjs5g5bQTnj8/Um6QiYVC5y3FXVFnDvGVbmJ9XTHl1HYP7J/GtC8fxpdOHMyQtOeh4InFB5S7HRUNTMwvXlTJn6Rb+vmkHBlw4oWWWfl6OZukikaZyly61paKGucu28HJeMTv21DGkfxL3zcjhhtOHMbi/ZukiXUXlLhFX39jMwvxS5i7dwvsbd5DQw7hwQhZfnjaCc3MytW26yHGgcpeIqKpt4N315SxYV8o768uorm1kaFoy91+cw/W5wxnUPynoiCLdispdOm3rrn0sXFfKwvxSlmyuoKHJGZjai8tPGsQVJw/mnHGapYsEReUuHeburN1WxYJ1pSxYV8q6kioAxmamcvvnRnPJpGymDB+gQheJAip3Oar6xmaWbK5gQWiGXrK7lh4Gp40cwCNXTOCiidmMyewTdEwROYTKXQ6zu6aBd9aXsSC/lHfXl7OnrpHkxATOGZfBty/O4cIJWQzs0zvomCJyFCp3AVp2LNq/3LK0oJKmZiejT2++MHkwF03M5uwTMnRWI5EYonLvZhqamimsqGFjaTUbSvewoayaT0qq+LR8LwDjsvrwjXPHcPGkbCYPS9OBukRilMo9TjU2NVNQUcOmslCJl1azsXQPm3fsoaHJATCD4QNSyMnuw8xpI7hoYjajMlIDTi4ikaByj3GNTc0UVtawsXRPy2y8rOVyc/le6puaDzxueHoyOVl9uWBCFuOy+pCT3ZcTsvqQ3EtLLSLxSOUeI6pqGyjcUUNh5V4Kduw9MBvfvGMv9Y3/KPFhA5LJye7LeeMzGZfVl5zsPpyQ1YeUXvpRi3QnnX7Fm9lw4AVgENAMPOvuT5vZ94CvA+Whhz7i7n8KN2i8c3cq9tZTWLGXwooaCipq2FKxl4KKGgor9rKzpuGgxw9NSyYnuw/n5mQeNBNP7a0SF5HwZu6NwP3uvsLM+gLLzWxB6L6fuPuPwo8XX5qbne1VtRRU7GXL/gKv3EvBjpYC31vfdOCxZjCkfzIjB6Zw2UmDGTUwhZEDUxg5MJUR6SkqcRE5qk43hLuXACWh69Vmlg8MjVSwWNTc7JTvqWPrrn1sO/BRS/HO/UVec9ASSmKCMXxAS2lPG50eKu+WAh82IJnePbUeLiKdE5Hpn5mNAqYCHwJnA/eY2S1AHi2z+51tPGcWMAtgxIgRkYjR5aprGyjZXXtYee+/XVpVe2BLlP369O7JsAHJjM1M5cIJWS3lnZ7KyIEpDElL1q76ItIlzN3bf9TRPoFZH+Bd4HF3f83MsoEdgAPfBwa7++1H+xy5ubmel5cXVo5w1DY0sbOmnl01DeyqaaCsunWB17Jt1z627tpHdW3jQc/r2cPI7pfE0LRkhqQlMSQtmSFpyQxNS2Zw6Ha/pMSAvisRiXdmttzdc9u6L6yZu5klAq8CL7r7awDuXtrq/l8Dfwjna3SUu1NT38SufQ3sqqlnd01D6HoDu/btL+7Q5b6G0P0tt+taLZW0NiAlkSFpyQxPT+GM0ekHynt/gWf27a2Zt4hEpXC2ljHgN0C+u/+41fjg0Ho8wLXAmvAiHtm6bVXcO++jA2XdervuQyUl9iAtuRdpKYn0T05kVEYKaclppKUmHhhPS265L6tfEkPSkrT5oIjErHDa62zgZmC1ma0MjT0CzDSzKbQsyxQA3wgr4VH0TerJCVl9Woo5pRdpyYmh8g6Vdco/ilvHRRGR7iTsNfdICHrNXUQkFh1tzV2nnBcRiUMqdxGROKRyFxGJQyp3EZE4pHIXEYlDKncRkTikchcRiUMqdxGROBQVOzGZWTlQGManyKDlYGXRKtrzQfRnjPZ8EP0Zoz0fKOOxGunumW3dERXlHi4zyzvSXlrRINrzQfRnjPZ8EP0Zoz0fKGMkaVlGRCQOqdxFROJQvJT7s0EHaEe054Pozxjt+SD6M0Z7PlDGiImLNXcRETlYvMzcRUSklZgudzO7zMzWm9kmM3so6DyHMrPhZvaOmeWb2VozuzfoTG0xswQz+8jMjsspEY+VmaWZ2Stm9kno3/LMoDO1Zmb/Evr5rjGzuWaWFAWZnjOzMjNb02os3cwWmNnG0OWAKMz4w9DPeZWZvW5madGUr9V9D5iZm1lGENk6ImbL3cwSgF8AlwOTaDkD1KRgUx2mEbjf3ScC04G7ozAjwL1AftAhjuJp4G13nwBMJoqymtlQ4FtArrufBCQANwabCoDngcsOGXsIWOTu44BFodtBes+Stg0AAALhSURBVJ7DMy4ATnL3U4ANwMPHO1Qrz3N4PsxsOHAxsOV4BzoWMVvuwDRgk7tvdvd6YB5wdcCZDuLuJe6+InS9mpZSGhpsqoOZ2TDg88D/BJ2lLWbWDziXlvP14u717r4r2FSH6Qkkm1lPIAXYFnAe3P09oPKQ4auB2aHrs4FrjmuoQ7SV0d3/4u6NoZtLgGHHPdg/srT1bwjwE+D/0XIq0agVy+U+FChqdbuYKCvO1sxsFDAV+DDYJIf5KS2/qEc+u3iwxgDlwG9DS0f/Y2apQYfaz923Aj+iZRZXAux2978Em+qIsvefvD50mRVwnvbcDrwVdIjWzOwqYKu7fxx0lvbEcrlbG2NR+T+pmfUBXgXuc/eqoPPsZ2ZXAmXuvjzoLEfREzgV+JW7TwX2EvxywgGhdeurgdHAECDVzL4SbKrYZ2aP0rKs+WLQWfYzsxTgUeC7QWfpiFgu92JgeKvbw4iCP4cPZWaJtBT7i+7+WtB5DnE2cJWZFdCyrHWhmf0u2EiHKQaK3X3/Xzyv0FL20eIi4DN3L3f3BuA14KyAMx1JqZkNBghdlgWcp01mditwJXCTR9e22mNp+U/849BrZhiwwswGBZrqCGK53JcB48xstJn1ouVNrDcDznQQMzNa1orz3f3HQec5lLs/7O7D3H0ULf9+f3X3qJp1uvt2oMjMxoeGZgDrAox0qC3AdDNLCf28ZxBFb/ge4k3g1tD1W4E3AszSJjO7DHgQuMrda4LO05q7r3b3LHcfFXrNFAOnhn5Ho07MlnvoTZd7gD/T8mKa7+5rg011mLOBm2mZEa8MfVwRdKgY9E3gRTNbBUwB/iPgPAeE/qJ4BVgBrKblNRX4HoxmNhdYDIw3s2IzuwN4ArjYzDbSsrXHE1GY8edAX2BB6PXyX1GWL2ZoD1URkTgUszN3ERE5MpW7iEgcUrmLiMQhlbuISBxSuYuIxCGVu4hIHFK5i4jEIZW7iEgc+v9zSqYEBRdevgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(score);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done after 199\n",
      "Done after 199\n",
      "Done after 199\n",
      "Average reward: 199.0\n"
     ]
    }
   ],
   "source": [
    "ag_r.Simulate(3, eps=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "ag_r.envs.envs[0].close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## REINFORCE with Baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "\n",
    "S = env.reset()\n",
    "ag_rb = Agent(env)\n",
    "ag_rb.env._max_episode_steps = 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "score2 = []\n",
    "counter = 0\n",
    "mysum = 0\n",
    "interval = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "for i_episode in range(200):\n",
    "    traj = traj = ag_rb.REINFORCE_baseline(alpha=0.0001, alpha_v=0.0001)\n",
    "    counter += 1\n",
    "    mysum += len(traj)\n",
    "    if counter%interval==0:\n",
    "        score2.append(mysum/interval)\n",
    "        mysum = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXzcVbn48c+ZNfuepkm6t+lKW1pKy74jUJaCiqLoBUF5ueGCXgHRi1d/et2vIC6gqIBeBEQBEdla9r0t3bekadomzb5nksx6fn98v9/JTDKTfZmkz/v14tXMN98kh2n6zDPPec45SmuNEEKIqcU20QMQQggx+iS4CyHEFCTBXQghpiAJ7kIIMQVJcBdCiCnIMdEDAMjLy9Nz5syZ6GEIIcSksmXLlgatdX6szyVEcJ8zZw6bN2+e6GEIIcSkopQ6HO9zUpYRQogpSIK7EEJMQRLchRBiChowuCul/qCUqlNK7Yq49hOl1D6l1A6l1D+UUlkRn7tdKVWmlNqvlLporAYuhBAivsFk7n8CLu517QXgBK31CuAAcDuAUmopcA2wzPyaXyul7KM2WiGEEIMyYHDXWr8KNPW69rzWOmA+fBuYYX68Afir1tqrtT4ElAFrR3G8QgghBmE0au43AP82Py4GjkZ8rtK81odS6ial1Gal1Ob6+vpRGIYQQgjLiIK7UuoOIAD8xboU47aYewprre/TWq/RWq/Jz4/Zgy+EEBOmosHD66UNEz2MYRt2cFdKXQdcBlyrezaFrwRmRtw2Azg2/OEJIcTEuO+1cr722LaJHsawDSu4K6UuBm4FrtBad0Z86ingGqWUWyk1FygB3h35MIUQYnx1+4J0+0MTPYxhG3D7AaXUw8A5QJ5SqhK4E6M7xg28oJQCeFtr/Vmt9W6l1KPAHoxyzRe01sGxGrwQQowVbzBEIDiFg7vW+mMxLt/fz/3fB74/kkEJIcRE8wdC+IOT9xhSWaEqhBAx+IIh/KHJm7lLcBdCiBj8wRBaQzA0ObN3Ce5CCBGDL2Bk7f5JWneX4C6EEDH4zHq7BHchhJhCejJ3KcsIIcSUYWXsk7UdUoK7EELEYGXuPgnuQggxdfRk7lKWEUKIKcPK3AOTtNddgrsQQsRglWN8AcnchRBiypDMXQghpiCr5i597kIIMUUEgiGsXQekz10IIaaIyIAumbsQQkwRVr0dpBVSCCGmjMiFS5K5CyHEFBEd3CVzF0KIKcEfWZaRVkghhJgaIjP3yPr7ZCLBXQgheomaUJWTmIQQYmqQCVUhhJiCImvuMqEqhBBTRGTmLod1CCHEFOGXsowQQkw9PinLCCHE1OOTvWWEEGLqkVZIIYSYgvyyiEkIIaYeK6ArNYW3H1BK/UEpVaeU2hVxLUcp9YJSqtT8M9u8rpRSdyulypRSO5RSq8dy8EIIMRaszD3FaZ/SW/7+Cbi417XbgI1a6xJgo/kY4BKgxPzvJuA3ozNMIYQYP14zc09xO6J63uPZX9NOtz841sMakgGDu9b6VaCp1+UNwAPmxw8AV0Zcf1Ab3gaylFKFozVYIYQYD+HM3TVw5t7tD3L5Pa/z2JbK8RjaoA235l6gta4GMP+cZl4vBo5G3FdpXutDKXWTUmqzUmpzfX39MIchhBAj960ndvLo5p7Q5QuEcNoVLrttwFZIrz+ELxCivds/1sMcktGeUFUxrsV82dNa36e1XqO1XpOfnz/KwxBCiMH75/ZqXi9tCD82grsNh9024CImvznhGkqwlsnhBvdaq9xi/llnXq8EZkbcNwM4NvzhCSHE2AqGNG3dfroiaub+YAiXw4bTrgbM3INmUE+0tU7DDe5PAdeZH18HPBlx/T/MrplTgFarfCOEEImotcuP1kRNiPqCIVx2G067bcBWSCv4h3RiZe6OgW5QSj0MnAPkKaUqgTuBHwKPKqVuBI4AV5u3PwOsB8qATuBTYzBmIYQYNc2dPgA6fRHBPaCNsoxNDViWsSZcJ11w11p/LM6nzo9xrwa+MNJBCSHEeGnpNCZCu3zRmbvbYcPlsNHhDfT79YFwWSaxgrusUBVCHNdazMw9quZuTaja1ICtkFbZJphgmbsEdyHEca05TuZuTKgO3AoZLstI5i6EEIkjZuYeNPrcBxXcp1i3jBBCTAnWhGpk5u4NGJm7wz6YCdXE7JaR4C6EOK5ZE6q+YCgcqI3M3WyFHCAl9ydot4wEdyHEcc0K7tBTmvEFjG4Zp13hH6CWHpRuGSGESDxWWQZ6gntk5j5QzT28/YBk7kIIkTiaIzN3X0/m7nLYcNhsA7dCBiVzF0KIhNPS6SPNbazn7MncjRWqTocacD/3YDhzH9txDpUEdyHEca2500dhZhLQk7lb3TJO2xAmVBMsuktwF0Ict7r9Qbr9IQqzkoGe4O6P2DgspPsvucgKVSGESDBWp0yRlbn7e9Xc7cYRFf1NqkrNXQghEozVKVOYaWTunb7IbhmF0wzugX4zd+lzF0KIhFDf7uUPrx+iyWME96Ksnsw9FNIEQhqX3Y7TboRIf6C/zN06iWmMBz1EEtyFEMedv22p5LtP7+Gf242D4orMmnu3PxjujnE6FA4ruPcTucN7y0jmLoQQE2tfTRsAf99aBRDulun09QR3l92GK1xz76csI90yQgiRGPZWG8HdCuRWzb3LFwyXYKxFTEC/7ZB+6ZYRQoiJ5w0EOVjv4YIlBQAkO+0ku+wkOW10+aMzd6fDLMv0E9yDCdotM+Axe0IIMZWU1XUQDGmuXFVEfXs3DR3GpGqy025m7kaQdtptOG0Dl2X8CdotI8FdCHFc2VfdDsDi6Rn85OqV1Ld7AUhxOczM3WiHNPrcrbJMfzX3xOyWkeAuhDiu7Ktpw+2wMSc3BYfdxsKCdACjLOML4ovM3M0J1f72lwlKt4wQQky8fTXtLCxID2fllmSXParm7jbPUIUBJlSlW0YIISbe3up2Fk9P73M9xemg0xcIT55a+7nDAK2Q0i0jhBATq6HDS0OHl8WFGX0+l+Sy0+UP4YtshbT63AexiEkydyGEmCA1rd0AzMxO7vO5ZKeN7ohFTE67wjWU7QcSK7bLhKoQ4vjR2mXsApmZ7OzzuRSXg05/ICpzt9sGsXGY9LkLIcTEsrb4zUzpG9yTnHa6fKFwzd1lt2GzDWLLX+lzF0KIiWVl7lnJrj6fS3HZ6fIFaO8OAEb3jFVqH9SEaoJl7lJzF0IcN/oryyQ7jVbIg3UduB02CjOTwxOqg2mFnFLdMkqpryqldiuldimlHlZKJSml5iql3lFKlSqlHlFK9X2JFEKICdDa5cdlt5Hk7Bv6kl12Qhr2VLcxPz8Nu01FtEIOvIhpynTLKKWKgS8Ba7TWJwB24BrgR8D/aq1LgGbgxtEYqBBCjFRrl4+MZCdKqT6fS3baAdhZ1UpJQRpAeIVqv3vLJGi3zEjLMg4gWSnlAFKAauA84G/m5x8ArhzhzxBCiFHR2uUnK8ZkKhiZO0B7d4AF+VZwHzhzT9RumWEHd611FfBT4AhGUG8FtgAtWuuAeVslUBzr65VSNymlNiulNtfX1w93GEIIEfaR377FI+8difv51i5/zHo7GBOqFitzdwziDNVggnbLjKQskw1sAOYCRUAqcEmMW2P+H2ut79Nar9Far8nPzx/uMIQQAoBmj493K5rYcrg57j39BfckZ09wXzDN2J7AaRs4c/dPwW6ZC4BDWut6rbUf+DtwGpBllmkAZgDHRjhGIYQYUHlDB0D40OtY+gvuVs3dYVPMzk0BwGZT2G1qUGWZ/jL3lk4fNz24meZ+xjbaRhLcjwCnKKVSlDE7cT6wB3gJ+LB5z3XAkyMbohBCDKy83gNAYz8BtKVz4LLM3LzUcK0djGDf737u4bJM/LHtOdbG83tq2XWsNf5No2wkNfd3MCZOtwI7ze91H3ArcItSqgzIBe4fhXEKIUS/yhuM4B4vcw+GNO3dgQHLMgumpUVdd9lt/e7nbvXA91eWsU5r6u8dwGgb0QpVrfWdwJ29LpcDa0fyfYUQYqjK682yTEfs4N7eHX8BE/R0y5T0Cu4Oe/+Z+2D63INmXd7XzwZko01WqAohpgSrLNPuDeANBPt8vr/VqQDTM5IomZbGWQujGzycdtvgJlT7qblbLw7ecQzusreMEGLSC4Y0hxs7yUhy0NYdoNnjZ3qmPeoea9OweH3uqW4HL9xydp/rRnAf2a6QgXBZZvw6aiRzF0JMelXNXfiCIU6anQ1Ao8fb556BMvd4nHYV3hwslsHsCmndI2UZIYQYgoNmG+SaOTlA7EnV4QZ3l8NGt79vmccymMM6rHt8McpFY0WCuxBiUnlpXx11bd1R1w6Z9fY1Zube5PFRWtvOd57aHZ7oHG5wT3Y56PT1F9wHX5bpr+tmtElwF0IkrE5fIOpxaW07n/rTe/z5negtBsobOshIclBSYKwsbezw8cS2Kv70ZgX1HUaJxgruGUMM7qkuO139BfeIoB6vYyYoNXchhDDsPtbK8u88T1ldR/jaH944BIDHGx30y+s9zMtPIyvZiU0ZmfvBOiObtyZSW7v8JDltUdsMDEaKy95/5h5Rj4/XMWOVZcazW0aCuxAiIe2vaScY0lSYi5MaO7w8vrUKgK5eNfBDDR7m5aVisymyU1w0enwcNPverYy9tZ/Vqf0xyjKBmJ/TWuMPahzmcXzxSjMyoSqEEKYas67e1GlMjv757SP4AiGSnNETnJ2+ANWt3czLTwUgJ9VFfXs3FY1W5m58fX/7yvQntZ/M3YrlbocRSuM1zFh1+fFcoSrBXQiRkOrajFq5FZxfOVDHSbOzKc5KxuvvCZLW4qV55h7sOakuth1tDde3W8zMvaXLF/Ps1IEk91Nzt4K12yz1xC3LSOYuhBCGmlYjc282a+aNHh/FWckkOe1Rmfshs2wzN8/I3HPTXDR09PS5t1llma7AkCdTway5+4PoGIHbCtouc6OxeGUZ2X5ACCFMte1mcDd71hs7fOSmuUgyD7K2lNd7UKonuOek9mTnSvVMqLYNsyyT4nIQDOmYk6FB892ByyzLxOuW8U9AWUa2HxBCJKTacObuo9sfpMMbIC/NTbLTHjXBWd7QQVFmcrgLJifVDUB+uptAMERLlw+tNY0eL9lxth7oj7UVcJcv2KfTxtpXxqq5xyvLWBm9V2ruQojjWSikqWs3SivNnf7witPcVJc5odoTJA81eMKTqdY9APPzU8lKcdHaFaDDG6DbH2JahnvIY7GCe2eMVaqBwWbuUpYRQgijvm7Vs1s6fTSa2/jmprlxO+10m8v4tdZGj3teT3DPCQf3NDKTnbR0+sIvFNPSk4Y8lhSXUeDo9PZthwz0ytzjLVINSreMEEJArdkGmZPqosnjp8HcCCwn1UWSw0632b1S3+6lwxsId8pY90BPcG/r8lNvBvf89BFk7jE6Znpn7tItI4QQ/bCC++Lp6VGZe16ai2SXjW4zSB4Mt0H2ZO4LC9KZnZvCaQtyyUpx0tLlj8jchx7ck/sL7mbm7nIY98QrywSkLCOEOB4dqG3n0feOhh/Xmj3ui6dnEAhpjpgLknLT3Ebmbta/e7dBgpGdv/Kf57J4egZZyU5aOkeWuadaZZkYq1StjDw8oTrg3jIS3IUQx5GH3jrMNx7fEQ7CNW3dKAULC4xyS1l9B26HjVSXPdznrrWmzmyXLMxMjvl9M5OdtHX7qW3rxmW3DbMVcuCyzEDdMv4JOIlJgrsQYsJZQf3VA/UA1LV1k5fmJi/NyLTL6jrITXWhlCLZZSekje1zPd4AyU47dnNvl94yU1xobZyvmp/uRqnY9/UnOaIVsjcrE3eFtx/oP3OXLX+FEMcVa1vel83gXtPWTUGGm+xUI9M+1OAh1wz0Vpbc7Q/h8QVJdcff5THLzNRL6zrIG0ZJBnrKMp4YZZlguCxjbj8QJ3bLhKoQ4rhkbRfwWmk9wZCmts3L9IwkslOMzhd/UJObZnxsLSTy+oN4vAFS3fHXYlplmCNNncOaTIX+J1T9vcsy8SZUzagvNXchxHGlvt0I5i2dfrZXtlDb1s20iOAOkGuuPLWCe7c/hMcbDPehx2Idhq318CZTwQjcNhW7LNPTLWP1uUsrpBBCAMbBG52+IBtWFWFT8JW/bqPJ42NmdgoZyU6sMrmVuSebwb3LzNzT+ivLRGw3MNzMXSlFqssRsywz1G4ZCe5CiOOGVZIpmZbOGSX5eLwBPnfOfK47bTZ2mwrXza1tBZKcVs09iMcX6Ddzj9wFcriZO8Tf9nfw3TJWWWb8jtmTjcOEEBPKCu55aS7+eP3JAFHdL9kpLpo7/eEJ1Z6yjJG5z8xOifu9I1sf89OGH9zjHbUXGEa3jNZ6WF07QyWZuxBiQkUuMLLbVJ+2Rqu00jOhambuAaPm3l+3jNthD/epT8sY+r4ylpQ4R+313c899tcHIjL28WqHlOAuhJhQ9ebWAvEya2tStacs09N3PlBZBnqy95GUZeJm7tbGYdZJTANsPwDjV5oZUXBXSmUppf6mlNqnlNqrlDpVKZWjlHpBKVVq/pk9WoMVQkw99e1elIo+ZCNStnm9d1nGG7AmVAcX3PPShn7EniU5TnC3ArWVucfrlokM+uM1qTrSzP0u4Fmt9WJgJbAXuA3YqLUuATaaj4UQIqaGDi85KS4c9tjhyDpgo3fm3uzxEdKQ0k9ZBoyyTlaKM7zQaDhS45RlwouYnP13y0Rm6+MV3Ic9oaqUygDOAq4H0Fr7AJ9SagNwjnnbA8DLwK0jGaQQYnJo7/bT6QtSMIT6dn27t9+SyeUri0h22sNBPcmcvGw0D/AYKHOfkZ0y4j1dBppQtV44BpO5j9dCppF0y8wD6oE/KqVWAluALwMFWutqAK11tVJq2siHKYSYDH763H5eK2tg09fOGfTXNHR4w3vIxLJiRhYrZmSFH1srRq3gnjpAzf3Oy5eOuM4drxXS3/skpnitkBE19/HaPGwkZRkHsBr4jdZ6FeBhCCUYpdRNSqnNSqnN9fX1IxiGECJRHG3uoqLBE85oB2OgzL23JDNLbjRbKPvrlgFIT3LGrecPVqrbETNzDw6yWyYY0uEun8lQc68EKrXW75iP/4YR7GuVUoUA5p91sb5Ya32f1nqN1npNfn7+CIYhhEgUTWYdvNZsbxyI1trM3AcffG02hctuCx/g0d/eMqMl2Wmnyx/scxhH+IDsAWrugaAOd/WMV1lm2MFda10DHFVKLTIvnQ/sAZ4CrjOvXQc8OaIRCiEmjeZOI+Aea+ka1P3WwdVDbVN0O23hssxArZCjweqV7+p1SHZgkN0ygVAovG3CePW5j/RZuRn4i1LKBZQDn8J4wXhUKXUjcAS4eoQ/QwgxSTR5Bh/cn3i/Klw/H2pwT3bawytbB5pQHQ0pbus0pmDUO4Wh7C2TkWQG90TvlgHQWm8D1sT41Pkj+b5CiMnHHwzR3m20Cx5r6e733rK6Dr7yyLbw4/4mVGNJctrD56JaWfVYSnFa2/4GgJ6xBoIhHDaFzVxVG3dCNajD45QVqkKIScUqyQBUt/afub9d3gjApSsKyU11sbAgfUg/y5qchHHK3OPs6R4IaRx2hU31H9yDIR1+lzIpMnchhLC0dPrDH1tlmbr2bvJS3eHM1vLOoSYKMtzc87FVwzv6ztmTrY/HhGpkWSZSIKhx2GzYzf+H+CcxhcJzA5OhW0YIIcKawhOcdqpauqlu7eKMH73EzQ+/H9UaqbXm7fJGTpmXO+zdEa29XJx2Fe4xH0s9mXv0KtVAKGRk7uYQenfThO8Ljn/mLsFdCDEqms3gvrQwg+rWLt462IgvEOJfO6v5+mPb8QaMrLe8wUN9u5dT5uUO+2dZq1XHI2uHnncKvTN3v5W5m+9MYu3nrrUmENLhun3Ct0IKIUSkJrPmvqwog5ZOPy/trycz2cnXP7CQJ7Yd49K7X+fdQ028U94EwLq5OcP+WdYWBAOtTh0t1otI71WqwVAIp12FyzKxau5WB814T6hKzV0IMSy9D52wMvdlRZkAvLCnhjMW5PPF80pYVpzJt/6xi4/c+xbpbgfT0t3MzUsd9s+2ShwDrU4dLVZg7n3UXiCosdtU+HmIVZax2iWtur2UZYQQCe2Ke97gFy8eCD9u8vhJddmZnWucjNTtD4Wz83MXTeP5r57FHeuXkJbkYP3ywhGdRmRtQTAeC5gAMpKMnSnbuqKDuz+kcdojyjIxgns4c59ki5iEEMehJo+PnVWtUQdQN3f6yE51UZSVHL62bl5P6SXV7eAzZ83jM2fNG/HPt1ohx6MNEox3CklOW1S7JxhlGYetpywTa38yaxWrtEIKIRLerqpWAA43doavNXl85KS6mJ6ZhFJGPXxpYcaY/PyeCdXxKcsA5KS4wh1BFr9ZlumvW8Y6hclpt+GwKQnuQojEtdMM7pXNnfgCIVwOI6vNTnHhtNuYnpHEwoL0uAdwjFQ4uI9TWQaME6GaewX3QDAUVZaJNaFq1dwdZtvmZNjPXQhxnLIy95CGqpYu5ual0uTxMT8/DYC7rlkVPtB6LIx3KyQYxwA29SrL9F6hGqsVMhzcbQqn3SYTqkKIxLXrWCuFmcZpSxWNHsBYoWodZr12bk440I8Fq+Y+0BF7oyk7JVbmro29ZfrplgkGreBuw+Wwyd4yQojE1NLp42hTF+uXFwJwuMGDNxCkwxsIn3c61qzMPW0cyzI5qX1r7oFQKHoRU4y4be357rAb+9D7AiM7FWqwJLgLIYZk97E2AM5ZlE+qy87hps7wvjLZIzzxaLCsFaMp41iWyU5x0dYdiKqZ95RljMexyjJWK6TdpiRzF0IkLmsy9YSiTGbnpnK4sTOc0Y70OLvB6mmFHMdumVTjXUlkO6RVllHKCPCxyjLWi4HDZjMz977H9Y0FCe5CiCHZUdnCjOxkslNdzM5NoaLRE65FWzX3seaegAlV611Js6dn90tvIBjeuMxuU/1uP+CwWd0yUpYRQiSYUEjz1sFG1s01Nv2anZvK0aZOatuNwznGK3NPnoBWyBzzhSuy7t7tD4Xr/0qp/rtl7Aqnffz63CW4CyEGbW9NG82dfk5fYAT3Obkp+IOa/3lmH7mpLmbmJA/wHUbH6lnZ3HLhQk6dP/ydJYcqnLl3Rgb3YHgrBLtSsRcx9e6WkeAuhEg0b5Q1AHD6gjzAyNzBaIO895MnjdteLy6HjS+dXxLOmseD9a4kOnMPhuv/dpuK2S1jrVA1JlTteGURkxBiPLR2+klx23EOYjXpG2WNzM9PpSDD6HFfUphOcVYyX79oIWvmDH8L38nA2kcnstfdG+gpy9hUnBWqZubutCtcdoVfMnchxFjTWnP+z1/hvlfLB7zXFwjx7qEmzjCzdoCsFBdv3HYeV62aMZbDTAhuh500tyO8SlVrjTcQCk/uDjShKq2QQohxU9/hpaHDy7ajLQPeu+1oC13+IKdFBPfjTXaqM5y5e80M3CrL2JSKueWvNaHqtFutkBLchRBjrKrZOMi6rK5jwHtfL2vAphjR8XiTXU6KiyZzwVa33+hXtyZUbXEyd+v8WLu5t4wcsyeEGHNVLUZwP9zoCZ9xGs+bZQ0sL84kM3l8thhIRJE7Q3b7rcy9p1umv8zd6nOXzF0IMeaszD2koaKhZ2/2tm5/VFeIxxtg29GW47okA9F7uocz90F2yzjs0gophBgnVuYO0aWZW/+2g2t//0748buHmgiEdNRk6vEoO9UV7nPvDljB3SrL9N8t47CZG4dJWUYIMdaqmo292JWKDu5bDjezt7qNo01GNv9GWQMuh42TZmdP1FATQk6qi05fkG5/MKIs0zOh2u/2A/aebhkd477RJsFdiONYVUsX8/PTKM5KpqzeCO717V7q2r0AvLS/DjAmU9fMzh7XRUOJyNo7p7nT12dCNV7N3R/ZCmm3oTXjsr+MBHchjlNaayqbu5iRnUzJtLRw5r6n2tjS125TbNpXR01rN/tq2sOrUo9n2eGFTP5wcHc7+++WCUbsCmm9OI5HaWbEwV0pZVdKva+Uetp8PFcp9Y5SqlQp9YhSanx2EhJCDElbV4AOb4DirGQWTEujvL6DYEizx9yvfcOJRbx1sJHb/r4Dl8PGpebhHMczaxfKTl+gT1lmwG4Zu8Jt3mu9MIyl0cjcvwzsjXj8I+B/tdYlQDNw4yj8DCHEKKtsMerpxdlGcPcGQlQ1d7H7WCszspO58sRivIEQL++v5xsXLWJOXuoEj3jipZr7x3d4A+HW0aSIzD12t0zPhKrb3B7YOw4dMyMK7kqpGcClwO/Nxwo4D/ibecsDwJUj+RlCiLFhtUHOyE5m0fQMwKit76luY2lhBmvn5pDudrBubg43nD53IoeaMHoy92BEK2TP3jKxJkp79nPvKcuMR+Y+0o3DfgF8A0g3H+cCLVrrgPm4EiiO9YVKqZuAmwBmzZo1wmEIIYbKaoMszkomJ9XFyplZ3LXxAHXtXjasLCbJaefJL57OtIwkbNY5csc5a/94jzeiLBNxWEes/dx7TmKKyNz9CZy5K6UuA+q01lsiL8e4Nea0sNb6Pq31Gq31mvz8/OEOQwgxTFXNXSQ5beSkulBKcdvFi6lt86I1LC0yMvl5+WmkjeNpR4kuxWVk3rEz99g192BIo5RRtrEmXwdaDTwaRlKWOR24QilVAfwVoxzzCyBLKWX9NswAjo1ohEKIUdfpC7C9soXirGSMaiqcOj+XcxYZiZYV3EU0qyzjiZpQ7X9XyEBI47QZodbK3LsTOXPXWt+utZ6htZ4DXANs0lpfC7wEfNi87TrgyRGPUggxLAdq2/tsCvbS/jrO++krvFfRzFWroqumP7hqOT/84HKKs8bnRKXJxu2wYbcpoywTCOK0K+xmySput0wwFL7H7ZgcmXs8twK3KKXKMGrw94/BzxBCDMLXH9vO9X98l4C5KvK3rxzkhj+9R1aKk8c/dypfPK8k6v6irGSuWStzYPEopUhx2fF4g1FH7Bmfg1CcbhmH3QjuSc7xy9xHpZimtX4ZeNn8uBxYOxrfVwgxfFpryuo66PQF+dfOajzeID/89z4uXVHITz68YtyOxJtqUl0OOn0B7DZbuIYORlkm1qZggaDGMQGZu/ztCrfu0ToAAB3ASURBVDFF1bZ56fQZQeSuF0upaevmjAV53POxVeE6uxi6FLcdjy+I267DmTjE75YJhDR2s+Zu3Z/Q3TJCiMRW3mDU2q88sYjyBg82pfjRh1dIYB+hNLeDTrPmbk2QgrlxWMxumRBOu2TuQohRcqjBA8BXL1yINxDiipVFMlE6Cqyau1IqaiO1uJl7UPdMqDrHb4WqBHchpqjyeg9JThszs1P4zSdOmujhTBmpLgc1bd047NHB3ehz73t/INRTc7cmYCfL3jJCiAR0qMHDnNxUWV06ylLcjvAipsiae7ztBwKhEA67cZ/TrlBqEuwtI4RIXIcaPMzLl82+Rlua2x7efiCyFdI4Zi92WcbK3JVSJDnskrkLIYbHHwxxpKmTeXlpEz2UKSfFZWbugWB0WSZOzT0Y0ecORt1dMnchxLAcaeokGNLMlW16R12qy25sP+ALhidIwVihGqtbxh/RCglG3V1aIYUQw3Ko3uiUkbLM6EtxO9Aamjv9g+qWCYZC4bIMGJl79yTdfkAIMc483gBn/ngTj2+pBHp63CVzH33W5mFdg9x+wB9RcwdjfxrJ3IUQg7JpXx1Hm7q4a2Mp/mCIx7dUUTItjawUOeVytKW6egJ6Uu+yzCBq7klOu2TuQojBeWZnNXab4khTJ199ZBv7a9v54nkLJnpYU1Lknjx9yjJxzlB1RNTcJXMXYorq8Ab42qPbwytIh6u1y8/uY610+gK8tL+Oa06eyZzcFJ7eUc2CaWlctqJolEYsIlnnqEJ05m6Lt597MLrmnuS0y/YDQkxFD711mMe3VlLX3s2DN6wd9l4vX3t0Gxv31XHJCdPp9oe4bEURS4syuOMfu/jS+SXhJe9idMXN3Ps5icneq+be2CHbDwgxpXT7g9z/+iHSkxy8VtpgHJyxuGDI32fPsTZe3FtHXpqbZ3bWkJfmYu3cHNbNzaFkWjonz8keg9ELIOrYwcgJVZsiZnD3B0M47ZFlmfHJ3KUsI8Q4emxLJQ0dXn597Wrm5aXy/57eSyDWhiS91LR209rpDz/+1ctlpLkdPPuVM/nU6XP46oULsdsUNpti7dwc2flxDKVETKi6e5VlYlRl+mbuTtvkOaxDCDE4f3zjECfOzOKMBXl8/twFfP2x7Ryo7ej3zNJQSPOh37xJIBTioRvXUdXcxTM7q/ns2fPJS3Nz5+XLxvH/QKS6+ynLxNnPPWqFqsMuu0IKMZW0d/spr/fwnxctQinFqllZAOw+1srSogz++u4RZuemcur83Kiv23KkmaqWLlx2G5f98nV8gRBz81L59BlzJ+J/47iXEtUKObS9ZYyvsUlZRkw9VS1dfPg3b1LT2j3RQxl3B81VoyXTjP1e5uSmkuy0s6e6jW5/kP96ajf/+bft+HuVaf61oxq3w8ZTN5/Ourk53HrxYp79ypnkprnH/f9BGBOiPVv4DqJbptf2A27ZfkBMRS/uqWXz4WbePNgw0UMZd2V1xqrRBWZwt9sUSwrT2X2sjR2VrfgCISqbu3hq27Hw1wRDmmd2VnPuomksnp7BQzeu43PnzA+f6CPGn3VINtBrP/fYE6qBiJOYwHhx8AVDMfehGU0S3MW4eq+iCYD9te2j9j2PtXRxxo82hb93oiqr68BltzErJyV8bWlRBnuPtfF2eSNgbBfwq5fLuPnh91n2X8/ypYffp67dy6UrCidq2CIGq+7eu+Ye0n33dA8GoydUra8Z67q7BPch6v2WeSrSWrOzsnVMMosth5sBOFAzesH9j28corK5i41760b0fbTWvHuoiS/8ZSu/feXgKI2uR1ldB3PzUsMHNwAsK8qk3RvgH+9XsaggnVsuXEh5vYfndtVw8twcntlVTZLTxnmLp436eMTw9WTu0WUZoE/HTCCke7VCWkftjW3dXSZUgTfLGghqzZkl+f3eV1bXwfq7X+P/Pr2ONXNyxml040trzff/tZffv36I7191Ateum93v/e9VNPHy/jq+eG4Jya7+SwVVLV1Ut3bjsCkO1HbEvMcbCA6p5NDe7eev7x4FYEdlCwCbK5qoaOzkwyfN6PdrvYEgP/jXXjasKmb1rGz+59/7uO/VcmwKnt1dwwVLCsIllN5f9+kHNnPp8kKuWTurz+d3VbXywp5ayuo6+OalS8LnlpbVtbOsKDPq3qWFRpfMoQYPnzxlNuuXF9LW7efUebnMy09jV1UrHm8gqkNDTLy0OJk7QFBrbPRk6oFQSDL3ifLdp/fw7Sd2DXjf37dW4guE2Ha0ZRxGNTH++597+P3rh7DbFK8dGLgu/uNn9/Grlw7ywd+8ydGmzvD1tm5/n3s3m2WTi5ZNp6qli/Ze92zaV8uq777AG2WDr8c/8t5R2r0BTpyZFX638aNn93H733dE9YXH8vvXDvHAW4f59AOb+fPbh7nv1XI+umYmr916HilOO//zzN6YX/fE+1W8VtrAt5/cFX5BsbxX0cSVv3qDX24q5d+7qrlnUylgLF460tTJ/F4vFoump4f/4a+dm4Pdprh23Wzm5Rv3nVCcybp50d0zYuJZq1SjFjGZf4+96+6RZ6hCT+Y+1qcxHffBvdsfpLSug4rGTurbvXHv01rzzx3GRFdkEJtMalq7+93PpLy+gz+9WcEnTpnFVauKeftQY5/SjDcQ5PN/2cLWI80ca+nivYpmLlxaQFVzJzc9tIVQSPOvHdWs+u4L4RfB6tYuKps72VzRTKrLzuUrjT1PIrP3+nYv//nYDjp9Qe57tXxQ/z+bK5r49csHWTsnh4+vnUW7N8C2yha2HG7GH9Q8v6cmfG8opHn/SHO4HlrV0sUvN5Vyyrwc/MEQ33piF4unp/PfG5ZRnJXMF89bwMZ9dX1eaIIhzb2vlLN4ejr5aW5ufvj98ItUbVs3n//LVmbmpPDeHRfwsbWzeHxLFXVt3VQ0eghp+rwTSHLaWWAG8rVzp+a7wanI2l8mahGTmblHdsx4vAG0js7wra+RzH2MHahtD7/Sbj3SHPe+bUdbONrUBRin3Iw144zG+K/svkCIPcfahvTq/5VH3ueT978T8xBfgD+9WYHLbuPL5y/ktPm5tHT62VvTFnXPtiMtPLOzhm8/sYt/bjde7O5Yv4TvXXkCe6vbePi9I/y/f+0hGNI8+GYFvkCIq3/7Fuf+9GWe2FbFqlnZLDMX7BwwJ1W11tz2+A7avQGuWFnEKwfqqWjw0Nrpp7HDeME92tTJWT9+KfyC8a8d1Xzsd2+TnuTgBx9czoqZRrnjnk1lhLRRC316R3V43A+8VcFVv36TuzeW0eULctvjOwD42UdO5NfXruaE4gzu/tiq8D/C606bQ1FmEj9/4UDU8/Xc7hrKGzzcfF4Jd31sFUebOrnjH7vo9AW46aEteLwBfvuJk8hNc3PTWfMIhELc/8YhSs0XspIYZZ5183JYWphBQUbSoP8uxcRKcTlQqicLB7DK6pGZe2mvDinoyfbHOnM/7gt5e471BK8th5u5aNn0mPc9tf0YLruNtXNzODzC4P7QWxU8tqWS2y5ZzGnz8/p8vtsf5PJ7XsfrD3HvJ09iYUE6tW3dzDS7LO56sZTfvFJGtz/E58+ZzzcuXhzz5/gCIf69q5oPLJ1OW7efdw41obXRqbJ4evSKyNZOP49truSKE4vIT3eHF9K8dbAxqk787iGjtLL7WBuHGjwsL85kTl4qs3JSuP/1Q3zriV1oDatnZfH0zmrmT0ujsrmLsxbm81ppPWeW5FGclUyKy85+c1L14XePsnFfHf912VIuW1HIMzur+faTu9hV1Uqq28HGr53NH944xJGmTh557wgrZ2Tyk+f2sbAgnf/79ClkpjgJhjQpLjub9tWRleLkI2tm8ofXD9Hs8ZGV4uThd4/gsCn+98UD/G3rUSqbu/j+lcspzkqmOCu5z3xLktPO586Zz7ef3M1bBxs5bUEerV1+fvLcfubmpXLxCdOx2xRfvWAhP3vhALuOtVLR4OG3nziJRdPTAZidm8r65YU8+OZhlhZlYFOxD8/49mVLCQTHti1OjK5UtwO3wxa1zUNP5t5z334zOVps/k6AZO7jZvexNtLcDlbPygrXhMEoUfzqpTK01oTMXuNzFuWzrCiDyqauQXeSNHt8dHgDUdce31rFjspWPv67d7g3RlfGXRtLKa/34A2E+OCv32T5d57jzB+/xCsH6qlv93LPS6WsmZ3DsqIMnt1tlB6qWrr489uHo7KG371Wzpf/uo1fbirlmZ3V4Vn8yK4SrTVbjzTz7Sd30eUPcsPpxqrHwsxk5ual8tbBRrp8QVq7jNLDuxVNlExLY1lRBp2+IJevNFr0bDbF7ZcsQWu4dHkhP/jgcnyBED95bj+rZ2XxwKdO5p1vns+NZ8zFZlOUFKRzoLadQw0evvf0Hs5YkMf1p81hWkYSF58wnddKG0h1O6hs7uK+V8p5bLNxwtBzu2vZeqSFisZOrjt1DpkpTsDoGT/BfBE6e2E+V6wsIhDSPLu7hm1HWzhQ28Gdly/lzJI8Wjx+7r9uDR9f13cyNNLVa2YyLd3NLzaW4vEG+Ooj2zja1MmPPrQiXCf//LkLOHVeLuX1Hv7flcv5QK/k4Pb1S1gzJ5sth5uZk5sa9fbc4rTbBpyMFonllHk5nL8kesM363ciMjbsr+kgyWljZnZP+2t4QnWMFzJN6sx9f0079756kB9ctTzmP5reqlq6+Nyft5Cb6uLylUVceWKxsfS7MINVs7L44xsVdPuNE81/uamMf7xfxbmLpqHR1LZ5uXBpAb5gCF8wRE1bN0VmF0SkPcfauGvjAX7+kRNxO4zl4m3dfj51+ly+cO58giHNrqpWPnX6HPbXtPPHNyr4zJnzwpMxe461cd+r5Vx90gxuu2QxP3vhAMlOO//eWc0vXjzAeYum4Q9q/nvDMl4vbeDOp3ZzsL6Dnzy7n2d31/D+kRZ+/OEV1LR188tNpTjtij++UcHs3BQWT0/H5bCxaV8dXzjXOMjhh8/u495XyrHbFJ84ZVbUHienzMvl8a2VrP3Bi6S6jAx66+FmrlpdzOUrivjmP3ay4cTi8P2nzs/l4c+cwvIZmaS5HayZnc3mw83ccqGx3H5aek/ZYVFBGv94v4qLfvEqyU47P716Zfg5uH39Ek6cmcUnTpnNDX96j5+9cACAL5w7n1+9dJD/enIXLruNi06IDqQrZmTybkUT5y6axrKiDBYVpPODZ/aypDCDFJedq1bP4OPrZtPlD0bt7BdPktPOZ8+ez3ef3sOyO58D4HtXnhBVG7fbFPf+x0nsr2nn5BgdVMVZyTx04zq2H23B5Tjuc6kpY8OJxVG/+9AT3CP3lzlQ287CgvTw7zaM34TqsIO7Umom8CAwHQgB92mt71JK5QCPAHOACuAjWuv4xewRaOjw8vetVZxQlMkNA+yzUdvWzcd/9zZNHT6aPD5ueXQ7rV1+9la389GTZ3LS7GzufbWcXVWtLCvK5HkzI960rza8dPjshfnhxTdHmjpjBvcH36rgud21PL61ksLMZKpaulg5M4u7N5bidthYNTOLQEhzVkk+y4szueXR7eysamXlTGOfkR8+u4/MZCd3XLqErBQXP7hqOWC8nf/WE7vYX9POGQvymJ+fhtth486ndvPAmxU8v6eGkmlpPL61krL6DnzmW74/XH8y1//xPfbVtPO1CxcSCGnu3lRKk8fHxr213PtKOdecPJPbL1kSzoIt65dP58ltVayelc0rB+r53tN78PiCnDwnh3Xzctn4tXP6/P9H7ovyzUuX8EZpA6cv6NvtsX55IWV1Hayalc2HVs9gemZP4C/OSubTZ84D4BsXL+bKX73BSbOzufm8Ev70RgW7j7Vx0bICMpOjx/uBZdN5+UA95yzKRynFHz51Mv9x/zu8e6iJq0+aEQ7ogwnslutOm0NumotjLd0UZiax4cS+B2BkJDljBvZI1t+vmLrCZZmIzH1fTTvnLoou+VmtvmNdlhlJ5h4Avqa13qqUSge2KKVeAK4HNmqtf6iUug24Dbh15EPt6/QFeZw2P5dfvVTGZSsLufPJ3SwtzODm80ui7guGNJ9+YDMN7V4e+vQ6Vs3M4hP3v8P//HsfvkCIZUUZrJ5t7H+9aV8dde1ePD4ju9u4r44kh53F09OZlpEU3qrzSGMnyU47pXUdXLGyCJfDRiAY4vk9tYAxOTknN5X8dDd/++ypXPv7d3h8ayX+YAilYPXsbLQ2Vq49v6eGlTOz2FHZwqsH6vnGxYv6nH159ZoZ3LOpjJq2bj5xitF7PiM7hSWFGTz41mHsNsUDN6zl2V01PLmtyug+uWgxZ5bk86HVxTy6uZL1KwrxeAPctbGUj//ubUrrOjh9QS7fu/KEqEUWljNL8tnz3YvRWvPB37zJX98z+skH29WxelY2q2fF3lf8nEXTOGfRwAtzTpyZxY8/vIITijJJcto5b0kB/9x+jCt7ZU3WuF685ezw4+KsZP722dO456Uyrj9tzqDG3JvdpvpkaELEYlPRmXtjh5eGDm94DsZiLXxK2FZIrXW11nqr+XE7sBcoBjYAD5i3PQBcOdJB9ufrFy2i0ePjwp+/yr931fDrlw/S1u3ncKOHz/9lCwdq2/nz24fZWdXKDz+0gtWzslFK8Z3Ll4VfYZcVZZKX5ubS5YXc92o5v9xURl6amxvOmMu2oy28V9HE2earb2FWEnab4nCTh689tp2vP7adC//3Fd4ub+S9imaaPD4uWlZAeb2HTfvq+PBJM3DabXxodTHl9R4efvcIi6dnkJnsJCvFxbq5OTy/23hB+NVLZWQkOfjkKX0XDrkddm5fv5izFuZzwZKeoHih+fHFy6ZTlJXMDWfM5ckvnsHmb13Ajea7mW9ftpQ/37iO+flpnFCUybz8VFq7/Nx4xlx+fe1JMQN7JKUUnzt7PgAzc5IpzOz7jmUsfWTNzHC56IbT53DRsgLOHeSKzexUF9++bGl4MlqIsdK7W8Zq9e0d3Mcrcx+VIqBSag6wCngHKNBaV4PxAgDE/FeolLpJKbVZKbW5vr5+2D979axsLlxaQKcvwJfOL6HLH+SJ96v43tN7eGZnDR/+zZv89Ln9nLEgj8si9ucoKUjn02fOIzvFGW5T+sEHl1OQkcTe6jYuW1HIB5YWoLWxCOFss5vCabdRlJXE0zuqKavr4OPrZmFTipse3Mz9r5eT5LTx4w+tJD/d2LHvo2tmAnDJ8kLcDhu1bd6oU3I+sLSA0roO7nxyF8/truX60+eSnhRdbrBsOLGYB29YG7V8/fKVReSkurjprHlxn6P0JCdnlBhdOTab4sWvns0bt57HN9cv6VPaiOeCJQWsnJnFhUtidxONl1Wzsrn3k2sGNccixHiyMner5G51yiwqiJ25J/z2A0qpNOBx4Cta67bBngCjtb4PuA9gzZo1I+oDu/uaVdS3e5mVm8JL++q468VSGj0+rjt1Nq+VNlDZ3MV/b1jW53SaWy9exM3nLQhPdGUmO7nn46u4+eH3+ejJM1k8PZ2CDDft3QFOigjIs3NSeb2sgVSXnTvWL6HJ4+Pye17nxb11Ri04xcm3Ll3C/pp25pitbxlJTj6wbDr/3H4sqj574bLpfO9fe/nzO0c4sySPG08f2h7dJQXpbP32hUP6msjJnaF8zROfP01O+BEiDnuvFar7azvITnGGEz2LO9znnrg1d5RSTozA/het9d/Ny7VKqUKtdbVSqhAY2W5Og5DssjMr13jbfe26Wdz2950UZiZx+/olfD0Yor7dG17O3Wv8ffbsWDUrm9dvPS/8+ObzSmjt8kftd2K9xb/ixCJS3Q5S3Q7uvmYVn3lwM1efZGTqseq01582h/01bZy+oKe3vTgrmee+chZ5aa4+dfZEI4FdiPgiu2WqWrrYuLeWpUUZff7dJPzGYcoY8f3AXq31zyM+9RRwHfBD888nRzTCIbp8ZRH/9+4RPnf2fJKcdpKc9rhljsH4RIz699w8I7h/9OSePumzFuaz/c4P9FsuOGl2Ns9/9ew+12NtTiWEmFysIN7Y4eP2v++gyx/kjvVL+9xnsylc9rE/R3UkmfvpwCeBnUqpbea1b2IE9UeVUjcCR4CrRzbEoUl1O3jqi2eM6c/46JpZzMlN5cRe7W1SBxbi+GXtCvn9Z/ZytLmLh25YG/dsXLdj7I/aG3Zw11q/DsR7n37+cL/vZJCZ4uyzElEIcXyz+hy2H23hk6fM7nc3T7fTPuaZuyyZE0KIUWB1yyjFgIsqxyNzl+AuhBCjwJpQvWBJQcwN4iK5nTa8gdCYLmSS4C6EEKNgWnoSDpvis2fHX3NiSXLYqW/3suGeN2JuHjgaJvXGYUIIkSiWz8hk53cuGtQOn26njXcPNeGwKZYXZw54/3BI5i6EEKNksFs3Wwd2/PBDKzhtQd8zHUaDZO5CCDHOrjttNlecWDTgIe4jIcFdCCHG2cUnFA580whJWUYIIaYgCe5CCDEFSXAXQogpSIK7EEJMQRLchRBiCpLgLoQQU5AEdyGEmIIkuAshxBSktB7R8aWjMwil6oHDw/zyPKBhFIcz1mS8Y0vGO7ZkvGNrqOOdrbXOj/WJhAjuI6GU2qy1XjPR4xgsGe/YkvGOLRnv2BrN8UpZRgghpiAJ7kIIMQVNheB+30QPYIhkvGNLxju2ZLxja9TGO+lr7kIIIfqaCpm7EEKIXiS4CyHEFDSpg7tS6mKl1H6lVJlS6raJHk9vSqmZSqmXlFJ7lVK7lVJfNq9/RylVpZTaZv63fqLHalFKVSildprj2mxey1FKvaCUKjX/zJ7ocQIopRZFPIfblFJtSqmvJNLzq5T6g1KqTim1K+JazOdTGe42f593KKVWJ8h4f6KU2meO6R9KqSzz+hylVFfE8/zbBBlv3L9/pdTt5vO7Xyl1UYKM95GIsVYopbaZ10f2/GqtJ+V/gB04CMwDXMB2YOlEj6vXGAuB1ebH6cABYCnwHeDrEz2+OGOuAPJ6XfsxcJv58W3AjyZ6nHF+H2qA2Yn0/AJnAauBXQM9n8B64N+AAk4B3kmQ8X4AcJgf/yhivHMi70ug5zfm37/5b2874AbmmvHDPtHj7fX5nwH/NRrP72TO3NcCZVrrcq21D/grsGGCxxRFa12ttd5qftwO7AWKJ3ZUw7IBeMD8+AHgygkcSzznAwe11sNd6TwmtNavAk29Lsd7PjcAD2rD20CWUmrsz2OLEGu8WuvntdYB8+HbwNgd/DlEcZ7feDYAf9Vae7XWh4AyjDgybvobr1JKAR8BHh6NnzWZg3sxcDTicSUJHDiVUnOAVcA75qUvmm9z/5AoZQ6TBp5XSm1RSt1kXivQWleD8YIFTJuw0cV3DdH/KBL1+YX4z+dk+J2+AePdhWWuUup9pdQrSqkzJ2pQMcT6+0/05/dMoFZrXRpxbdjP72QO7irGtYTs61RKpQGPA1/RWrcBvwHmAycC1RhvxRLF6Vrr1cAlwBeUUmdN9IAGopRyAVcAj5mXEvn57U9C/04rpe4AAsBfzEvVwCyt9SrgFuD/lFIZEzW+CPH+/hP6+QU+RnSCMqLndzIH90pgZsTjGcCxCRpLXEopJ0Zg/4vW+u8AWutarXVQax0Cfsc4vzXsj9b6mPlnHfAPjLHVWuUB88+6iRthTJcAW7XWtZDYz68p3vOZsL/TSqnrgMuAa7VZEDbLG43mx1swatgLJ26Uhn7+/hP5+XUAHwQesa6N9PmdzMH9PaBEKTXXzNyuAZ6a4DFFMWto9wN7tdY/j7geWUe9CtjV+2snglIqVSmVbn2MMZG2C+N5vc687TrgyYkZYVxRGU+iPr8R4j2fTwH/YXbNnAK0WuWbiaSUuhi4FbhCa90ZcT1fKWU3P54HlADlEzPKHv38/T8FXKOUciul5mKM993xHl8cFwD7tNaV1oURP7/jOVM8BjPP6zE6UA4Cd0z0eGKM7wyMt307gG3mf+uBh4Cd5vWngMKJHqs53nkY3QTbgd3WcwrkAhuBUvPPnIkea8SYU4BGIDPiWsI8vxgvOtWAHyNzvDHe84lRNviV+fu8E1iTIOMtw6hVW7/DvzXv/ZD5e7Id2ApcniDjjfv3D9xhPr/7gUsSYbzm9T8Bn+1174ieX9l+QAghpqDJXJYRQggRhwR3IYSYgiS4CyHEFCTBXQghpiAJ7kIIMQVJcBdCiClIgrsQQkxB/x8rl/d7lBc87QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(score2);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.0245504   0.0344018  -0.00279386  0.02541287]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([1.8533], grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S = ag_rb.env.reset()\n",
    "print(S)\n",
    "ag_rb.V(torch.tensor(S, dtype=torch.float))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([17.9258], grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ag_rb.V(torch.tensor([0, 0, 0, 0], dtype=torch.float))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}