GridWorld.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GridWorld env"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## To do:\n",
    "- "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "from copy import deepcopy\n",
    "import gym\n",
    "from gym import spaces\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 699,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GridWorld(gym.Env):\n",
    "    \n",
    "    def __init__(self, n, goal=None, holes=None):\n",
    "        '''\n",
    "         env = GridWorld(n, goal, holes=None)\n",
    "         \n",
    "         Creates a 2D GridWorld environment of size (n,n).\n",
    "         \n",
    "         Coors are (row,col).\n",
    "         Actions are indexed 0-3\n",
    "              2\n",
    "            3 + 1\n",
    "              0\n",
    "         goal is a list of tuples for the goal state ([row,col], r)\n",
    "        '''\n",
    "        super().__init__()\n",
    "        self.n = n\n",
    "        n_actions = 4\n",
    "        self.P = np.array([[1,0],[0,1],[-1,0],[0,-1]], dtype=int)\n",
    "        self.action_space = spaces.Discrete(n_actions)\n",
    "        self.observation_space = spaces.MultiDiscrete((self.n,self.n)) #spaces.Discrete(self.n**2)\n",
    "        self.terminal = []\n",
    "        self.R = np.zeros(self.observation_space.nvec)\n",
    "        if goal is not None:\n",
    "            for g in goal:\n",
    "                self.terminal.append(self.c2i(g[0]))\n",
    "                self.R[g[0][0], g[0][1]] = g[1]\n",
    "        else:\n",
    "            self.terminal.append(self.c2i([3,4]))\n",
    "            self.R[3, 4] = 10.\n",
    "        self.coords = self.observation_space.sample()\n",
    "        \n",
    "    def seed(self, seed_val):\n",
    "        super().seed(seed_val)\n",
    "        \n",
    "    def reset(self):\n",
    "        self.coords = self.observation_space.sample()\n",
    "        return self.coords\n",
    "    \n",
    "    def state(self):\n",
    "        return self.coords\n",
    "    \n",
    "    def step(self, action):\n",
    "        s = self.coords + self.P[action,:]  # Proposed next state\n",
    "        # Is it a valid state? What is the reward?\n",
    "        if s[0]<0 or s[0]>=self.n:\n",
    "            reward = -4.\n",
    "            s[0] = np.clip(s[0], 0, self.n-1)\n",
    "        elif s[1]<0 or s[1]>=self.n:\n",
    "            reward = -4.\n",
    "            s[1] = np.clip(s[1], 0, self.n-1)\n",
    "        else:\n",
    "            self.coords = s\n",
    "            reward = self.R[tuple(self.coords)]\n",
    "        done = self.c2i(self.coords) in self.terminal\n",
    "        return self.coords, reward, done, {}\n",
    "    \n",
    "    def c2i(self, coords):\n",
    "        return coords[0]*self.n + coords[1]\n",
    "    def i2c(self, idx):\n",
    "        row = idx//self.n\n",
    "        col = idx%self.n\n",
    "        return np.array([row,col])\n",
    "    \n",
    "    def flatten(self, x):\n",
    "        return np.reshape(x, (self.n**2, x.shape[-1]))\n",
    "    def unflatten(self, x):\n",
    "        return np.reshape(x, (self.n, self.n, x.shape[-1]))\n",
    "    \n",
    "    def __str__(self):\n",
    "        blah = f'Location: {self.coords}'\n",
    "        return blah\n",
    "    def __repr__(self):\n",
    "        return self.__str__()\n",
    "        \n",
    "    def showR(self):\n",
    "        print(np.flipud(self.R.T))\n",
    "        \n",
    "    def draw(self, traj=None, fig=None):\n",
    "        '''\n",
    "         fig = env.Draw(traj=None, fig=None)\n",
    "         \n",
    "         Draws the arena, along with a single trajectory(optional).\n",
    "         traj is a list of dictionaries, where each element is:\n",
    "           dict(a=<action>, s=<state>, sp=<next state>, r=<reward>)\n",
    "        '''\n",
    "        tcoords = []\n",
    "        if traj is not None:\n",
    "            #tcoords.append(traj[0]['s'])\n",
    "            for t in traj:\n",
    "                tcoords.append(t['s'])\n",
    "                if t['d']:\n",
    "                    break\n",
    "            tcoords.append(traj[-1]['sp'])\n",
    "            tcoords = np.array(tcoords)\n",
    "        if fig is None:\n",
    "            fig = plt.figure(figsize=(8,8))\n",
    "        for row in range(self.n):\n",
    "            for col in range(self.n):\n",
    "                r = self.R[row,col]\n",
    "                if r>0:\n",
    "                    plt.plot(col, row, 'yo', markersize=r)\n",
    "                elif r<0:\n",
    "                    plt.plot(col, row, 'ko', markersize=-r)\n",
    "        if traj is not None:\n",
    "            plt.plot(tcoords[:,1], tcoords[:,0], 'o--')\n",
    "        plt.grid(True)\n",
    "        #plt.plot(traj[0]['s'][0], traj[0]['s'][1], 'go')\n",
    "        plt.plot(tcoords[0,1], tcoords[0,0], 'go')\n",
    "        plt.plot(tcoords[-1,1], tcoords[-1,0], 'ro')\n",
    "        #plt.axis('image')\n",
    "        plt.axis([-1,self.n,self.n,-1])\n",
    "        return fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 700,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2 5] [0 4]\n",
      "[2 1] [5 5]\n"
     ]
    }
   ],
   "source": [
    "e1 = GridWorld(6)\n",
    "e2 = GridWorld(6) #deepcopy(e1)\n",
    "e2.seed(314)\n",
    "e3 = deepcopy(e1)\n",
    "e3.seed(112)\n",
    "print(e2.coords, e3.coords)\n",
    "e2.reset()\n",
    "e3.reset()\n",
    "print(e2.coords, e3.coords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 701,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Vector_env(gym.Wrapper):\n",
    "    '''\n",
    "     class Vector_env\n",
    "     \n",
    "     Acts as a wrapper for the OpenAI gym environments.\n",
    "    '''\n",
    "    def __init__(self, EnvConstructor, n=1):\n",
    "        '''\n",
    "         venv = Vector_env(env, n=1)\n",
    "         \n",
    "         Creates n copies of env.\n",
    "        '''\n",
    "        super().__init__(EnvConstructor())\n",
    "        self.envs = []\n",
    "        self.n_envs = n\n",
    "        for k in range(n):\n",
    "            env_r = EnvConstructor() #deepcopy(env)\n",
    "            r = np.random.randint(10000)\n",
    "            #print(id(env_r), r)\n",
    "            env_r.seed(r)\n",
    "            self.envs.append(env_r)\n",
    "        self.observation_space = self.envs[0].observation_space\n",
    "        self.action_space = self.envs[0].action_space\n",
    "    \n",
    "    def reset(self):\n",
    "        '''\n",
    "         states = venv.reset()\n",
    "         \n",
    "         Resets all the environments, and returns an array with the\n",
    "         reset states in each row.\n",
    "        '''\n",
    "        states = []\n",
    "        for e in self.envs:\n",
    "            state = e.reset()\n",
    "            states.append(state)\n",
    "        return np.array(states)\n",
    "    \n",
    "    def c2i(self, c):\n",
    "        i = np.zeros(len(c), dtype=int)\n",
    "        for idx,cc in enumerate(c):\n",
    "            i[idx] = self.envs[0].c2i(cc)\n",
    "        return i\n",
    "    def i2c(self, i):\n",
    "        c = np.zeros((len(i), 2), dtype=int)\n",
    "        for idx,ii in enumerate(i):\n",
    "            c[idx,:] = self.envs[0].i2c(ii)\n",
    "        return c\n",
    "    \n",
    "    def step(self, actions):\n",
    "        '''\n",
    "         S, R, done, info = venv.step(actions)\n",
    "         \n",
    "         Takes one step in each of the environments, and returns an array\n",
    "         for each of the resulting outputs.\n",
    "         \n",
    "         Inputs:\n",
    "           actions   an array of actions\n",
    "           \n",
    "         Outputs:\n",
    "           S    array of states after the step\n",
    "           R    array of rewards from the step\n",
    "           done array of Boolean flags:\n",
    "                  False means episode continues\n",
    "                  True means episode is done\n",
    "        '''\n",
    "        S, R, dones = [], [], []\n",
    "        for e,a in zip(self.envs, actions):\n",
    "            next_s, r, done, inf = e.step(a)\n",
    "            if done:\n",
    "                S.append(e.reset())\n",
    "            else:\n",
    "                S.append(next_s)\n",
    "            R.append(r)\n",
    "            dones.append(done)\n",
    "        return np.array(S), np.array(R), np.array(dones), {}\n",
    "    \n",
    "    def state(self):\n",
    "        S = []\n",
    "        for e in self.envs:\n",
    "            S.append(e.state())\n",
    "        return np.array(S)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 702,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBuffer():\n",
    "    def __init__(self, size=1000000):\n",
    "        self.memory = [] #deque(maxlen=size)\n",
    "        \n",
    "    def store(self, s, a, r, sp, d):\n",
    "        self.memory.append((s, a, r, sp, d))\n",
    "    \n",
    "    def sample(self, num=32):\n",
    "        num = min(num, len(self.memory))\n",
    "        return random.sample(self.memory, num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 703,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AgentBase(object):\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def choose_action(self, s):\n",
    "        raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 704,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent(AgentBase):\n",
    "    '''\n",
    "     class Agent(env)\n",
    "     \n",
    "     Represents an RL agent. Its Q function is represented by a neural network, so that\n",
    "       agent.Q(s) returns a vector of probabilities over action-space.\n",
    "       \n",
    "     The input env is a sample of the environment that the agent will act in.\n",
    "    '''\n",
    "    def __init__(self, venv):\n",
    "        '''\n",
    "         ag = Agent(venv)\n",
    "         \n",
    "         Instantiates an Agent object.\n",
    "         \n",
    "         Inputs:\n",
    "           venv  a Vector_env object, containing the environments for the task\n",
    "        '''\n",
    "        self.gamma = 0.9\n",
    "        self.state_shape = venv.observation_space.nvec\n",
    "        self.n_states = np.prod(self.state_shape)\n",
    "        self.n_actions = venv.action_space.n  #n_actions\n",
    "        \n",
    "        self.venv = venv  # useful for accessing states, etc.\n",
    "        \n",
    "        # Q function, states x actions\n",
    "        self.Q = np.random.normal(size=(self.n_states, self.n_actions)) / 10.\n",
    "    \n",
    "    def choose_action(self, s, eps=0.):\n",
    "        '''\n",
    "         A = ag.choose_action(s, eps=0.)\n",
    "         \n",
    "         Choose an action, given the state s.\n",
    "         With probability eps, it will choose an action randomly.\n",
    "         Otherwise, it will choose an action that yields the highest Q-value.\n",
    "        '''\n",
    "        s_flat = self.venv.c2i(s)\n",
    "        Qvals = self.Q[s_flat,:]\n",
    "        a = np.argmax(Qvals, axis=1)  # greedy actions\n",
    "        \n",
    "        bs = len(s)\n",
    "        \n",
    "        if eps>=1.e-8:\n",
    "            a_random = np.random.choice(self.n_actions, size=(bs,))\n",
    "            r = np.random.rand(bs)\n",
    "            ridx = r<eps\n",
    "            a[ridx] = a_random[ridx]\n",
    "            \n",
    "        return a\n",
    "    \n",
    "    def update(self, s, a, r, sp, d, alpha=0.01):\n",
    "        '''\n",
    "         env.update(s, a, r, sp, d, alpha=0.01)\n",
    "         \n",
    "         Learn from a bunch of transitions.\n",
    "         This method updates the Q-function.\n",
    "        '''\n",
    "        si = self.venv.c2i(s)\n",
    "        Qsa = self.Q[si,a]\n",
    "        spi = self.venv.c2i(sp)\n",
    "        Qsp = self.Q[spi]  # value for all actions\n",
    "        maxQs = np.max(Qsp, axis=1)\n",
    "        delta = alpha * (r + self.gamma*maxQs - Qsa)\n",
    "        self.Q[si,a] += delta\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 705,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(agent, venv, T=10000, batch_size=32, alpha=0.01, eps=0.):\n",
    "    '''\n",
    "     ag.train(T=10000, batch_size=32, alpha=0.01, eps=0.)\n",
    "\n",
    "     Trains the agent on the environ\n",
    "    '''\n",
    "    # First, let's populate a replay buffer\n",
    "    buf = ReplayBuffer(size=T)\n",
    "    s = venv.reset()\n",
    "    for k in range(T):\n",
    "        a = agent.choose_action(s, eps=eps)\n",
    "        sp, r, done, _ = venv.step(a)\n",
    "        buf.store(s, a, r, sp, done)\n",
    "        s = sp\n",
    "\n",
    "        for batch in buf.sample(batch_size):\n",
    "            agent.update(*batch)\n",
    "    return buf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 706,
   "metadata": {},
   "outputs": [],
   "source": [
    "def episode(agent, env, s=None, max_iters=100, eps=0.):\n",
    "    done = False\n",
    "    n_iters = 0\n",
    "    if s is None:\n",
    "        s = env.reset()\n",
    "    else:\n",
    "        env.coords = np.array(s)\n",
    "    traj = []\n",
    "    while not done and n_iters<max_iters:\n",
    "        a = agent.choose_action([s], eps=eps)\n",
    "        sp, r, done, _ = env.step(a[0])\n",
    "        #print(s, a, r, sp, done)\n",
    "        traj.append(dict(a=a[0], s=s, sp=sp, r=r, d=done))\n",
    "        s = sp\n",
    "        n_iters += 1\n",
    "    print(n_iters, done)\n",
    "    return traj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 707,
   "metadata": {},
   "outputs": [],
   "source": [
    "goal = [([1,1], 10), ([6,7], 15), ((1,2), -10)]\n",
    "env = GridWorld(8, goal=goal)\n",
    "venv = Vector_env((lambda :GridWorld(8, goal=goal)), n=16)\n",
    "ag = Agent(venv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 708,
   "metadata": {},
   "outputs": [],
   "source": [
    "# s = venv.reset()\n",
    "# a = ag.choose_action(s, eps=0.3)\n",
    "# sp, r, done, _ = venv.step(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 722,
   "metadata": {},
   "outputs": [],
   "source": [
    "buf = train(ag, venv, T=1024, alpha=0.1, eps=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 731,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7 True\n",
      "[6 2]\n",
      "[5 2]\n",
      "[5 3]\n",
      "[5 4]\n",
      "[6 4]\n",
      "[6 5]\n",
      "[6 6]\n",
      "[6 7]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeUAAAHWCAYAAABJ3pFhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dfXRddb3n8c83aUJT0tNKb8A+oEhGgnAplNYqMrjS4hWqcC+XBUscYY1eZ+KV0YsMgjCz1GG5rs+CXjsLRUAd4KKlFkaQBx/qGQqrUGgLLaVNaQr00T5QT5vQh6TNd/5oCk3T0iRnt/u793m/1uqyOXtnn9/XpLxz9jn7xNxdAAAgfVVpLwAAAOxFlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIMqKspldbmZLzKzbzCYltSgAACpRuY+UX5R0qaQnElgLAAAVbUg5n+zuSyXJzJJZDQAAFYznlAEACOKwj5TN7I+S3nmQTf/T3f9vf+/IzFoktUjS0KFDJ77rXe/q9yKj6u7uVlVVtn+uycMMUj7myMMMEnNEkocZpHzMsXz58s3u3nC4/Q4bZXf/SBILcvfbJd0uSU1NTd7a2prEYVNVLBbV3Nyc9jLKkocZpHzMkYcZJOaIJA8zSPmYw8xe689+2f7RAwCAHCn3kqh/NLM1ks6R9DszezyZZQEAUHnKffX1A5IeSGgtAABUNE5fAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAgikSib2YVm1mpmK8zsxiSOCQBApSk7ymZWLel/S5om6TRJnzSz08o9LgAAlSaJR8qTJa1w95Xu3inpV5L+IYHjAgBQUczdyzuA2WWSLnT3/9Lz8VWSPuDuXzhgvxZJLZLU0NAwccaMGWXdbwQdHR2qr69PexllycMMUj7myMMMEnNEkocZpHzMMWXKlPnuPulw+w1J4L7sILf1Kb273y7pdklqamry5ubmBO46XcViUVmfIw8zSPmYIw8zSMwRSR5mkPIzR38kcfp6jaQT9/t4nKR1CRwXAICKkkSUn5X0XjN7j5nVSrpC0m8TOC4AABWl7NPX7r7bzL4g6XFJ1ZLucvclZa8MAIAKk8RzynL3RyQ9ksSxAACoVLyjFwAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiXIF27GjT8uVXa86cgqSpmjOnoOXLr9aOHW1pL21A2tradPXVV6tQKGjq1KkqFAq6+uqr1daWrTkAYJ9Eomxmd5nZRjN7MYnj4ch5/fVH9eyz47Vu3R3as6ddkmvPnnatW3eHnn12vF5//dG0l9gvjz76qMaPH6877rhD7e3tcne1t7frjjvu0Pjx4/Xoo9mYAwD2l9Qj5V9IujChY+EI2bGjTUuWXKbu7u2Sug7Y2qXu7u1asuSy8I+Y29radNlll2n79u3q6uo9R1dXl7Zv367LLruMR8wAMieRKLv7E5K2JHEsHDmrV/9A3d0Hxri37u4urV5961Fa0eD84Ac/6BPjA3V1denWW2PPAQAH4jnlCrJhwz3q+wj5QF3asOHuo7GcQbvnnnv6FeW77449BwAcyNw9mQOZnSTpYXf/20Nsb5HUIkkNDQ0TZ8yYkcj9pqmjo0P19fVpL2MApkrqz9fbJM0+wmsZvKlTp6o/37dmptmz485xoOx9Px0cc8SRhxmkfMwxZcqU+e4+6XD7HbUo76+pqclbW1sTud80FYtFNTc3p72Mfpszp9Dz4q63V11d0HnnbT0KKxqcQqGg9vbDz1EoFLR1a9w5DpS176dDYY448jCDlI85zKxfUeb0dQU54YQrJdUcZq8anXDCVUdjOYN25ZVXqqbm7eeoqanRVVfFngMADpTUJVH3SZorqcnM1pjZZ5M4LpJ14onXqarq7WNWVVWjE0+89iitaHCuu+66fkX52mtjzwEAB0rq1defdPfR7l7j7uPc/c4kjotk1dU16vTTZ6qqapj6PmKuUVXVMJ1++kzV1TWmsbx+a2xs1MyZMzVs2LA+ca6pqdGwYcM0c+ZMNTbGngMADsTp6wozatQ0vf/9izRmTIuqqwuSTNXVBY0Z06L3v3+RRo2alvYS+2XatGlatGiRWlpaVCgUZGYqFApqaWnRokWLNG1aNuYAgP0NSXsBOPrq6hp1yinTdcop01UsFnXeec1pL2lQGhsbNX36dE2fPj0XLwQBAB4pAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEGUHWUzO9HM/mxmS81siZldk8TCAACoNEMSOMZuSde5+wIzGy5pvpn9wd1fSuDYAABUjLIfKbv7endf0PP3dklLJY0t97gAAFSaRJ9TNrOTJE2Q9EySxwUAoBKYuydzILN6Sf9P0r+6+6yDbG+R1CJJDQ0NE2fMmJHI/aapo6ND9fX1aS+jLHmYQcrHHHmYQWKOSPIwg5SPOaZMmTLf3Scdbr9EomxmNZIelvS4u99yuP2bmpq8tbW17PtNW7FYVHNzc9rLKEseZpDyMUceZpCYI5I8zCDlYw4z61eUk3j1tUm6U9LS/gQZAAAcXBLPKZ8r6SpJU83s+Z4/H0vguAAAVJSyL4ly9yclWQJrAQCgovGOXgAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQZQdZTMbambzzOwFM1tiZjcnsTAAACrNkASOsUvSVHfvMLMaSU+a2aPu/nQCxwYAoGKUHWV3d0kdPR/W9Pzxco8LAEClSeQ5ZTOrNrPnJW2U9Ad3fyaJ4wIAUEls7wPdhA5mNlLSA5K+6O4vHrCtRVKLJDU0NEycMWNGYveblo6ODtXX16e9jLLkYQYpH3PkYQaJOSLJwwxSPuaYMmXKfHefdLj9Eo2yJJnZ1yW94e7fP9Q+TU1N3tramuj9pqFYLKq5uTntZZQlDzNI+ZgjDzNIzBFJHmaQ8jGHmfUrykm8+rqh5xGyzKxO0kckLSv3uAAAVJokXn09WtIvzaxaeyM/w90fTuC4AABUlCRefb1I0oQE1gIAQEXjHb0AAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABJFYlM2s2swWmtnDSR0TAIBKkuQj5WskLU3weAAAVJREomxm4yR9XNIdSRwPAIBKlNQj5R9KukFSd0LHAwCg4pi7l3cAs4skfczdrzazZklfdveLDrJfi6QWSWpoaJg4Y8aMsu43go6ODtXX16e9jLLkYQYpH3PkYQaJOSLJwwxSPuaYMmXKfHefdLj9kojytyRdJWm3pKGSCpJmufuVh/qcpqYmb21tLet+IygWi2pubk57GWXJwwxSPubIwwwSc0SShxmkfMxhZv2Kctmnr939Jncf5+4nSbpC0uy3CzIAADg4rlMGACCIIUkezN2LkopJHhMAgErBI2UAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCGJHEQM3tVUrukPZJ2u/ukJI4LAEAlSSTKPaa4++YEjwcc1oML1+p7j7dqbWmHxj49W9df0KRLJoxNe1kDkocZpPzMAaQpySgDR9WDC9fqplmLtaNrjyRpbWmHbpq1WJIyE4M8zCDlZw4gbUlF2SX93sxc0k/d/faEjgsc0vceb30zAvvs6NqjG2Yu0n3zVulrF5+m08eM0JMvb9aPZ7/c5/O/eekZamyo1x9f2qCfzVnZZ/utnzhLY0bW6aEX1umep1/rs/22KyfquGNrdf9zqzVz/po+23/xmcmqq63W3XNf1cOL1vfZ/uvPnXPIGb7ym0Vvxuzf/vSynlrR+yTUO4bV6idXTZQkfeexZVrw2l97bR89Yqh+eMUESdLNDy3RS+u29dp+csOx+tal4yVJN81apJWb3ui1/bQxBX394tMlSV/61UKt37qz1/az3/0OfeXCUyVJ/3z3fM1etlGde7r7zPG9x1uJMjAASUX5XHdfZ2bHS/qDmS1z9yf238HMWiS1SFJDQ4OKxWJCd52ejo6OzM+R5RnWlnYc9PbOPd0qlUp67rnntKlQrSWb96hU6uyz37xn5ml1fZUWb9ytUqmrz/a5c+dqVF2VXlp/8O1PPfWUhtealq3pUqm0u8/2J+Y8oWOqTctXHXx7sVg85Ay7dne/+XV55ZVOlUq9w717u725fdVrfbdX7Xxr+5o1u1Ta1juY67q2qVjcsvfv63ap9Ebv7Wu6t6lY3CRJ2rBhp0o7vdf2VWpXsfgXSdKmzTv7BHmftaUdmf3+yvK/jX3yMIOUnzn6w9z98HsN5IBm/0tSh7t//1D7NDU1eWtra6L3m4Zisajm5ua0l1GWLM9w7rdnHzRqY0fW6akbp6awooHLwwxSfubYX5b/beyThxmkfMxhZvP78yLosi+JMrNjzWz4vr9L+qikF8s9LnA411/QpLqa6l631dVU6/oLmlJa0cDlYQYpP3MAaUvi9PUJkh4ws33H+3d3fyyB4wJva99zlTfMXKTOPd0aO7Iuc6/43bfWN1+1nMEZpPzMAaSt7Ci7+0pJZyawFmDALpkwVvfNW6VSqaTHv5LN06SXTBirSyaMzfwpuksmjNUDC9fquCGdeujL2fxaAGnjHb0AJGZn1x517kn2dSpAJSHKAAAEQZQBAAiCKCPzvnbxafpP76tNexkAUDbeZhOZd/qYEdpUqD78jjjizn/f8Wpr60h7GUBm8UgZmffky5u1ZPOew++II67lw42a9p6atJcBZBZRRub9ePbL+m1b37fRBICs4fQ1gMR84qdzVSrtUIYvtwZSxSNlAACCIMoAAARBlAEACIIoI/O+eekZ+vTpx6S9DAAoG1FG5jU21Gt0Pd/KEVw0frQmj+b1o8Bg8V8yZN4fX9qghRt3p70MSLrqnJN0/ru4ThkYLKKMzPvZnJV67JWutJcBSTs692gXvyUKGDSiDCAxn/75PN3y3M60lwFkFlEGACAIogwAQBBEGQCAIIgyMu/WT5yllvFcpwwg+4gyMm/MyDqNquNbOYLLJo7TfxzLdcrAYPFfMmTeQy+s0zPruU45gssnnajzxnGdMjBYRBmZd8/Tr2n2Kq5TjmDLG51q7+Q6ZWCwiDKAxHz+nvmavpDrlIHBIsoAAARBlAEACIIoAwAQBFFG5t125UR9YcLQtJcBAGXjgkJk3nHH1mp4raW9DEi68oPv1ksvvZH2MoDM4pEyMu/+51ZrzhouiYrg4jPH6AOj+VkfGCz+9SDzZs5fo1KJNw+JYF1ph17f0Z32MoDM4pEygMRc++vndfuiXWkvA8gsogwAQBBEGQCAIIgyAABB8EIvZN4vPjNZT8x5Iu1lAEiQu2vbtrnatm2epEVavfp5FQqTVSicI7P8XgJJlJF5dbXVOqY6v/9Is+S/nneyFr+4OO1lIMO6u7u0fv2dWr36u+rs3Cj3LkmdWrmyVmY1qq09XieeeINGj/6sqqry92tCiTIy7+65r2r5qi41p70Q6COnnaAhG5emvQxk1O7dHVq8eJra2xeou3t7r23unXLv1M6dr6it7Tpt3PjvOuOMRzRkSH1Kqz0yeE4ZmffwovWat57rlCNo29Sh9R1cp4yB6+7u0uLF07Rt27N9gtx33+3atm2eFi/+mLq78/XGQYlE2cxGmtlMM1tmZkvN7JwkjgsgW/7HrMX6xRKuU8bArV9/p9rbF8i9f98/7rvU3j5f69ffdYRXdnQl9Uj5R5Iec/dTJZ0pifNXOCqufWi6Zq2/RL/f8XENvfkEXfvQ9LSXVLEeXLhWC1eV1PrXbp377dl6cOHatJc0KA8uXKtzvz1bn37sjczOkbUZ3F2rV3+3zyPk0v2n6pRLRunDU0ynXDJKpftP7bW9u3u7Vq/+rtz9aC73iCo7ymZWkPRhSXdKkrt3unup3OMCh3PtQ9P1o/lf1m7bKJlrlzbqR/O/TJhT8ODCtbpp1mJ17tl76nptaYdumrU4fAwOtG+OtaUdkrI5RxZn2LZtrjo7N/a6rXT/qfro7Ss1ZuvrqpJrzNbX9dHbV/YJc2fnBm3bNvdoLveISuKFXidL2iTp52Z2pqT5kq5xd35VDI6o2xZ8Q269T3W57dJtC76hWy/+Qkqrqkzfe7xVO7r29LptR9ce/esjS3XfvFV99r/hwiZNfPdxmv/aFn33sdY+27928Wk6fcwIPfnyZv149st9tn/z0jPU2FCvP760QT+bs7LP9ls/cZbGjKzTQy+s0z1Pv9Zn+21XTtRxx9bq/udWa+b8NW/evnBV6c0fLPaf44aZi3rN8evP7X2G7vYn2vSnpb1jMrSmWr/8p8mSpH/708t6asXmXtvfMaxWP7lqoiTpO48t04LX/tpr++gRQ/XDKyZIkm5+aIleWret1/aTG47Vty4dL0m6adYirdzU+z+1L67betCvxb4Zzn73O/SVC/eG7Z/vnq+/bu/ste+5/+Fv9C/nv1eS9J/vmqedBxzr/Pcdr5YPN0qSPvHTvjG8aPxoXXXOSdrRuUef/vm8PtsvmzhOl086UVve6NTn75kvSdq1a6127vyq3F1T3/WIPjB6jibfu0nDdvde27DdnZp87yYtv/yt29x3q739WY0Y8aE+95VFSUR5iKSzJX3R3Z8xsx9JulHSV/ffycxaJLVIUkNDg4rFYgJ3na6Ojo7Mz5HlGXb5JukgV0Lt8k2ZnCnLX4t9j8oOtKl9l44b0veFOAsWLFT7K9V6+a97VCp19tn+3HPPaVOhWks2H3z7vGfmaXV9lRZv3K1Sqe/x586dq1F1VXpp/cG3P/XUUxpea1q2pqvXLzM5MMj7314qvXUCcN/Xqe2Vrj6/DKW22t7c/sornSqVekdt9/a3tq96re/2qp1vbV+zZpdK23qvaV3XNhWLW/b+fd0uld7ovf2NXW8/wyq1q1j8iyRp0+ad6ujsfer3lVfaVSzufVS9ZctOde7pvb2trUPF7tWSpNJBvu7LX+5Qcder2rXHVSrt7LN92bIOFTva1N65//Y3JPW+n3du3XLQOd65dYuW7/exe6dWrFikFSuKB90/a6zcc/Fm9k5JT7v7ST0fnyfpRnf/+KE+p6mpyVtb+/50nDXFYlHNzc1pL6MsWZ5h6M0naJc29rn9GB2vnV/fkMKKypPlr8W535590DCPHVmnp26cmsKKBicPc2RxhtWrf6iVK78i97d+ADvlklEas/X1PvuuGzFKyx9863azY9TY+B2NG3fNUVnrYJnZfHefdLj9yn5O2d3/Imm1mTX13HS+pJfKPS5wOJ8/+6syP6bXbebH6PNnf/UQn4Ej5foLmlRXU93rtrqaal1/QdMhPiOmPMyRxRkKhcky6/1GIPM+1aDtQ2p73bZ9SK3mfaqh121mQzR8+PuP+BqPlqTePOSLku41s1pJKyV9JqHjAoe073nj2xZ8Q7t8k46xBn1+4ld5PjkFl0wYK2nvc8trSzs0dmSdrr+g6c3bsyIPc2RxhkLhHNXWHq+dO19587aRly/T73WqJt+7Se/cukV/GXGc5n2qQSMvX9brc2trT1ChkJ+rcMs+fT0YnL6OIw8zSPmYIw8zSMwRSZZmWLv2J2pru+6wbxyyv6qqYWpsvEVjx37uCK4sGUft9DUAAOUaPfqzGj78bJkdc/idtfe55OHDJ2r06H86wis7uogyACB1VVU1OuOMR1UoTFZV1bDD7DtMhcJknXHGI7n7pRREGQAQwpAh9TrzzD+psfEWDR16sqqqju155GwyO0ZVVcdq6NCT1dh4i84880+5+2UUEr8lCgAQSFVVjcaO/ZzGjGnRtm1z1d7+rFasWKTGxvEaPnyyCoUP8vuUAQA4msxMI0Z8SCNGfEgrVhQ1blxz2ks6Kjh9DQBAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQZQdZTNrMrPn9/uzzcy+lMTiAACoJEPKPYC7t0o6S5LMrFrSWkkPlHtcAAAqTdKnr8+X1OburyV8XAAAcs/cPbmDmd0laYG7Tz/IthZJLZLU0NAwccaMGYndb1o6OjpUX1+f9jLKkocZpHzMkYcZJOaIJA8zSPmYY8qUKfPdfdLh9kssymZWK2mdpNPdfcPb7dvU1OStra2J3G+aisWimpub015GWfIwg5SPOfIwg8QckeRhBikfc5hZv6Kc5Onradr7KPltgwwAAA4uySh/UtJ9CR4PAICKkkiUzWyYpL+TNCuJ4wEAUInKviRKktx9u6RRSRwLAIBKxTt6AQAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEkUiUzexaM1tiZi+a2X1mNjSJ4wIAUEnKjrKZjZX0L5ImufvfSqqWdEW5xwUAoNIkdfp6iKQ6MxsiaZikdQkdFwCAilF2lN19raTvS1olab2kre7++3KPCwBApRlS7gHM7B2S/kHSeySVJN1vZle6+z0H7NciqaXnw11m9mK59x3A30janPYiypSHGaR8zJGHGSTmiCQPM0j5mKOpPzuVHWVJH5H0irtvkiQzmyXpQ5J6Rdndb5d0e88+z7n7pATuO1V5mCMPM0j5mCMPM0jMEUkeZpDyMYeZPdef/ZJ4TnmVpA+a2TAzM0nnS1qawHEBAKgoSTyn/IykmZIWSFrcc8zbyz0uAACVJonT13L3r0v6+gA+JS/RzsMceZhByscceZhBYo5I8jCDlI85+jWDufuRXggAAOgH3mYTAIAgUouymV3e89ac3WaWqVfVmdmFZtZqZivM7Ma01zMYZnaXmW3M8qVpZnaimf3ZzJb2fC9dk/aaBsPMhprZPDN7oWeOm9Ne02CZWbWZLTSzh9Ney2CZ2atmttjMnu/vK2YjMrORZjbTzJb1/Bs5J+01DYSZNfV8Dfb92WZmX0p7XYMxkLeiTu30tZm9T1K3pJ9K+rK7Z+Kb38yqJS2X9HeS1kh6VtIn3f2lVBc2QGb2YUkdkv5Pz9ujZo6ZjZY02t0XmNlwSfMlXZLBr4VJOtbdO8ysRtKTkq5x96dTXtqAmdl/lzRJUsHdL0p7PYNhZq9q79sGZ/q6WDP7paQ57n6HmdVKGubupbTXNRg9/91dK+kD7v5a2usZiJ63on5S0mnuvsPMZkh6xN1/cbD9U3uk7O5L3b01rfsvw2RJK9x9pbt3SvqV9r55Sqa4+xOStqS9jnK4+3p3X9Dz93btvRRvbLqrGjjfq6Pnw5qeP5l7sYeZjZP0cUl3pL2WSmdmBUkflnSnJLl7Z1aD3ON8SW1ZC/J++v1W1DynPHBjJa3e7+M1ymAI8sbMTpI0QdIz6a5kcHpO+z4vaaOkP/Rcapg1P5R0g/aeAcsyl/R7M5vf806EWXSypE2Sft7zdMIdZnZs2osqwxWS7kt7EYMx0LeiPqJRNrM/9pxDP/BP5h5Z7scOclvmHuDNGaUAAAICSURBVNXkiZnVS/qNpC+5+7a01zMY7r7H3c+SNE7SZDPL1FMKZnaRpI3uPj/ttSTgXHc/W9I0Sf+t56merBki6WxJt7n7BElvSMrq619qJf29pPvTXstgHPBW1GMkHWtmVx5q/0SuUz4Ud//IkTx+StZIOnG/j8eJ34qVmp7nYH8j6V53n5X2esrl7iUzK0q6UFKWXoR3rqS/N7OPSRoqqWBm97j7If/jE5W7r+v5341m9oD2PmX1RLqrGrA1ktbsd8ZlpjIaZe394WiBu29IeyGD1K+3ot6H09cD96yk95rZe3p+grtC0m9TXlNF6nmB1J2Slrr7LWmvZ7DMrMHMRvb8vU57/xEvS3dVA+PuN7n7OHc/SXv/TczOYpDN7NieFw2q53TvR5WtH44kSe7+F0mrzWzfL0E4X1KmXgC5n08qo6euewzorajTvCTqH81sjaRzJP3OzB5Pay0D4e67JX1B0uPa+3/sDHdfku6qBs7M7pM0V1KTma0xs8+mvaZBOFfSVZKm7nfZxMfSXtQgjJb0ZzNbpL0/9P3B3TN7SVHGnSDpSTN7QdI8Sb9z98dSXtNgfVHSvT3fV2dJ+mbK6xkwMxumvVe6ZPYs2EDfipp39AIAIAhOXwMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACOL/AxGc8C4s9TgMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#traj = episode(ag, env, s=[1,3], eps=0.)\n",
    "traj = episode(ag, env, eps=0.)\n",
    "for t in traj:\n",
    "    print(t['s'])\n",
    "print(traj[-1]['sp'])\n",
    "\n",
    "env.draw(traj=traj);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 661,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[24.97125039, 18.3305528 , 18.0013044 , 20.06725828]])"
      ]
     },
     "execution_count": 661,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ag.Q[venv.c2i([[0,7]]),:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}