again_Cartpole.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cartpole, again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import random\n",
    "import time\n",
    "from tqdm import tqdm\n",
    "from copy import deepcopy\n",
    "import gym\n",
    "from gym import spaces\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "source": [
    "# Task"
   ],
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "env = gym.make('CartPole-v0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "env = gym.make('LunarLander-v2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "class GridWorld(gym.Env):\n",
    "    \n",
    "    def __init__(self, n, goal=None, holes=None):\n",
    "        '''\n",
    "         env = GridWorld(n, goal, holes=None)\n",
    "         \n",
    "         Creates a 2D GridWorld environment of size (n,n).\n",
    "         \n",
    "         Coors are (row,col).\n",
    "         Actions are indexed 0-3\n",
    "              2\n",
    "            3 + 1\n",
    "              0\n",
    "         goal is a list of tuples for the goal state ([row,col], r)\n",
    "        '''\n",
    "        super().__init__()\n",
    "        self.n = n\n",
    "        n_actions = 4\n",
    "        self.P = np.array([[1,0],[0,1],[-1,0],[0,-1]], dtype=int)\n",
    "        self.action_space = spaces.Discrete(n_actions)\n",
    "        self.observation_space = spaces.MultiDiscrete((self.n,self.n))\n",
    "        self.terminal = []\n",
    "        self.R = np.zeros((self.n, self.n))\n",
    "        if goal is not None:\n",
    "            for g in goal:\n",
    "                self.terminal.append(self.c2i(g[0]))\n",
    "                self.R[g[0][0], g[0][1]] = g[1]\n",
    "        else:\n",
    "            self.terminal.append(self.c2i([3,4]))\n",
    "            self.R[3, 4] = 10.\n",
    "        self.coords = self.sample_input()\n",
    "        \n",
    "    def sample_input(self):\n",
    "        return self.observation_space.sample()\n",
    "    \n",
    "    def seed(self, seed_val):\n",
    "        super().seed(seed_val)\n",
    "        \n",
    "    def reset(self):\n",
    "        self.coords = self.sample_input()\n",
    "        return self.coords\n",
    "    \n",
    "    def state(self):\n",
    "        return self.coords\n",
    "    \n",
    "    def step(self, action):\n",
    "        s = self.coords + self.P[action,:]  # Proposed next state\n",
    "        # Is it a valid state? What is the reward?\n",
    "        if s[0]<0 or s[0]>=self.n:\n",
    "            reward = -4.\n",
    "            s[0] = np.clip(s[0], 0, self.n-1)\n",
    "        elif s[1]<0 or s[1]>=self.n:\n",
    "            reward = -4.\n",
    "            s[1] = np.clip(s[1], 0, self.n-1)\n",
    "        else:\n",
    "            self.coords = s\n",
    "            reward = self.R[tuple(self.coords)]\n",
    "        done = self.c2i(self.coords) in self.terminal\n",
    "        return self.coords, reward, done, {}\n",
    "    \n",
    "    def c2i(self, coords):\n",
    "        return coords[0]*self.n + coords[1]\n",
    "    def i2c(self, idx):\n",
    "        row = idx//self.n\n",
    "        col = idx%self.n\n",
    "        return np.array([row,col])\n",
    "    \n",
    "    def flatten(self, x):\n",
    "        return np.reshape(x, (self.n**2, x.shape[-1]))\n",
    "    def unflatten(self, x):\n",
    "        return np.reshape(x, (self.n, self.n, x.shape[-1]))\n",
    "    \n",
    "    def __str__(self):\n",
    "        blah = f'Location: {self.coords}'\n",
    "        return blah\n",
    "    def __repr__(self):\n",
    "        return self.__str__()\n",
    "        \n",
    "    def showR(self):\n",
    "        print(np.flipud(self.R.T))\n",
    "        \n",
    "    def draw(self, traj=None, fig=None):\n",
    "        '''\n",
    "         fig = env.Draw(traj=None, fig=None)\n",
    "         \n",
    "         Draws the arena, along with a single trajectory(optional).\n",
    "         traj is a list of dictionaries, where each element is:\n",
    "           dict(a=<action>, s=<state>, sp=<next state>, r=<reward>)\n",
    "        '''\n",
    "        tcoords = []\n",
    "        if traj is not None:\n",
    "            #tcoords.append(traj[0]['s'])\n",
    "            for t in traj:\n",
    "                tcoords.append(t['s'])\n",
    "                if t['d']:\n",
    "                    break\n",
    "            tcoords.append(traj[-1]['sp'])\n",
    "            tcoords = np.array(tcoords)\n",
    "        if fig is None:\n",
    "            fig = plt.figure(figsize=(8,8))\n",
    "        for row in range(self.n):\n",
    "            for col in range(self.n):\n",
    "                r = self.R[row,col]\n",
    "                if r>0:\n",
    "                    plt.plot(col, row, 'yo', markersize=r)\n",
    "                elif r<0:\n",
    "                    plt.plot(col, row, 'ko', markersize=-r)\n",
    "        if traj is not None:\n",
    "            plt.plot(tcoords[:,1], tcoords[:,0], 'o--')\n",
    "        plt.grid(True)\n",
    "        #plt.plot(traj[0]['s'][0], traj[0]['s'][1], 'go')\n",
    "        plt.plot(tcoords[0,1], tcoords[0,0], 'go')\n",
    "        plt.plot(tcoords[-1,1], tcoords[-1,0], 'ro')\n",
    "        #plt.axis('image')\n",
    "        plt.axis([-1,self.n,self.n,-1])\n",
    "        return fig"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Vector_env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "class Vector_env(gym.Wrapper):\n",
    "    '''\n",
    "     class Vector_env\n",
    "     \n",
    "     Acts as a wrapper for the OpenAI gym environments.\n",
    "    '''\n",
    "    def __init__(self, EnvConstructor, n=1):\n",
    "        '''\n",
    "         venv = Vector_env(env, n=1)\n",
    "         \n",
    "         Creates n copies of env.\n",
    "        '''\n",
    "        super().__init__(EnvConstructor())\n",
    "        self.envs = []\n",
    "        self.n_envs = n\n",
    "        for k in range(n):\n",
    "            env_r = EnvConstructor() #deepcopy(env)\n",
    "            r = np.random.randint(10000)\n",
    "            env_r.seed(r)\n",
    "            self.envs.append(env_r)\n",
    "        self.observation_space = self.envs[0].observation_space\n",
    "        self.action_space = self.envs[0].action_space\n",
    "    \n",
    "    def reset(self):\n",
    "        '''\n",
    "         states = venv.reset()\n",
    "         \n",
    "         Resets all the environments, and returns an array with the\n",
    "         reset states in each row.\n",
    "        '''\n",
    "        states = []\n",
    "        for e in self.envs:\n",
    "            state = e.reset()\n",
    "            states.append(state)\n",
    "        return np.array(states)\n",
    "    \n",
    "    def step(self, actions):\n",
    "        '''\n",
    "         S, R, done, info = venv.step(actions)\n",
    "         \n",
    "         Takes one step in each of the environments, and returns an array\n",
    "         for each of the resulting outputs.\n",
    "         \n",
    "         Inputs:\n",
    "           actions   an array of actions\n",
    "           \n",
    "         Outputs:\n",
    "           S    array of states after the step\n",
    "           R    array of rewards from the step\n",
    "           done array of Boolean flags:\n",
    "                  False means episode continues\n",
    "                  True means episode is done\n",
    "        '''\n",
    "        S, R, dones = [], [], []\n",
    "        for e,a in zip(self.envs, actions):\n",
    "            next_s, r, done, inf = e.step(a)\n",
    "            if done:\n",
    "                S.append(e.reset())\n",
    "            else:\n",
    "                S.append(next_s)\n",
    "            R.append(r)\n",
    "            dones.append(done)\n",
    "        return np.array(S), np.array(R), np.array(dones), {}\n",
    "    \n",
    "    def set_max_episode_steps(self, n):\n",
    "        for env in self.envs:\n",
    "            env._max_episode_steps = n\n",
    "            \n",
    "    def state(self):\n",
    "        S = []\n",
    "        for e in self.envs:\n",
    "            S.append(e.state())\n",
    "        return np.array(S)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBuffer():\n",
    "    def __init__(self, size=1000000):\n",
    "        self.memory = [] #deque(maxlen=size)\n",
    "        \n",
    "    def store(self, s, a, r, sp, d):\n",
    "        for trans in zip(s,a,r,sp,d):\n",
    "            self.memory.append(trans)\n",
    "            #self.memory.append((s, a, r, sp, d))\n",
    "    \n",
    "    def sample(self, num=32):\n",
    "        num = min(num, len(self.memory))\n",
    "        blah = random.sample(self.memory, num)\n",
    "        s,a,r,sp,d = zip(*blah)\n",
    "        return np.array(s), np.array(a), np.array(r), np.array(sp), np.array(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'GridWorld' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-fa837c5dcf99>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mGridWorld\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'GridWorld' is not defined"
     ]
    }
   ],
   "source": [
    "env = GridWorld(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent(object):\n",
    "    '''\n",
    "     class Agent(env)\n",
    "     \n",
    "     Represents an RL agent. Its Q function is represented by a neural network, so that\n",
    "       agent.Q(s) returns a vector of probabilities over action-space.\n",
    "       \n",
    "     The input env is a sample of the environment that the agent will act in.\n",
    "    '''\n",
    "    def __init__(self, venv):\n",
    "        '''\n",
    "         ag = Agent(venv)\n",
    "         \n",
    "         Instantiates an Agent object.\n",
    "         \n",
    "         Inputs:\n",
    "           venv  a Vector_env object, containing the environments for the task\n",
    "        '''\n",
    "        self.gamma = 0.99\n",
    "        \n",
    "        #***** USE THIS FOR GridWorld *****\n",
    "        #self.state_shape = np.prod(venv.observation_space.nvec)\n",
    "        \n",
    "        #***** USE THIS FOR CartPole *****\n",
    "        self.state_shape = venv.observation_space.shape[0]\n",
    "        \n",
    "        self.n_actions = venv.action_space.n  #n_actions\n",
    "        \n",
    "        self.venv = venv  # useful for accessing states, etc.\n",
    "        \n",
    "        # Q function: R^4 -> R^2 Q-value vector\n",
    "        self.Q = torch.nn.Sequential(\n",
    "                      torch.nn.Linear(self.state_shape, 128),\n",
    "                      torch.nn.Tanh(),\n",
    "                      torch.nn.Linear(128, 32),\n",
    "                      torch.nn.Tanh(),\n",
    "                      torch.nn.Linear(32, self.n_actions),\n",
    "                      #torch.nn.LogSoftmax(dim=1),\n",
    "                    )\n",
    "        self.optim = torch.optim.Adam(self.Q.parameters(), lr=0.001)\n",
    "\n",
    "    def process_inputs(self, s):\n",
    "        return s   #***** FOR CartPole *****\n",
    "        #***** BELOW FOR GridWorld one-hot *****\n",
    "        blah = np.zeros((len(s), np.prod(self.venv.observation_space.nvec)))\n",
    "        for k,ss in enumerate(s):\n",
    "            q = int(self.venv.envs[0].c2i(ss))\n",
    "            blah[k,q] = 1.\n",
    "        return blah\n",
    "    \n",
    "    def choose_action(self, s, eps=0.):\n",
    "        '''\n",
    "         A = ag.choose_action(s, eps=0.)\n",
    "         \n",
    "         Choose an action, given the state s.\n",
    "         With probability eps, it will choose an action randomly.\n",
    "         Otherwise, it will choose an action that yields the highest Q-value.\n",
    "        '''\n",
    "        with torch.no_grad():\n",
    "            Qvals = self.Q(torch.tensor(self.process_inputs(s), dtype=torch.float))\n",
    "            a = torch.argmax(Qvals, dim=1).numpy()\n",
    "        \n",
    "        # epsilon-greedy\n",
    "        bs = len(s)\n",
    "        if eps>=1.e-8:\n",
    "            a_random = np.random.choice(self.n_actions, size=(bs,))\n",
    "            r = np.random.rand(bs)\n",
    "            ridx = r<eps\n",
    "            a[ridx] = a_random[ridx]\n",
    "            \n",
    "        return a\n",
    "    \n",
    "    def update(self, s, a, r, sp, d, alpha=0.01):\n",
    "        '''\n",
    "         env.update(s, a, r, sp, d, alpha=0.01)\n",
    "         \n",
    "         Learn from a bunch of transitions.\n",
    "         This method updates the Q-function.\n",
    "        '''\n",
    "        loss_fcn = torch.nn.MSELoss()\n",
    "                \n",
    "        # Get estimate of current value\n",
    "        Qs = self.Q(torch.tensor(self.process_inputs(s), dtype=torch.float))\n",
    "        Qsa = Qs[range(len(Qs)),a]\n",
    "        \n",
    "        # Bellmann's equation to get a better estimate\n",
    "        with torch.no_grad():\n",
    "            Qsp = self.Q(torch.tensor(self.process_inputs(sp), dtype=torch.float))  # value for all actions\n",
    "            maxQs = torch.amax(Qsp, dim=1)\n",
    "            target = torch.tensor(r, dtype=torch.float) + self.gamma*(1-torch.as_tensor(d).float())*maxQs\n",
    "            \n",
    "        # Gradient descent of Q(S,A) -> r + gamma argmax_s Q(S',a)\n",
    "        loss = loss_fcn(Qsa, target)\n",
    "        self.optim.zero_grad()\n",
    "        loss.backward()\n",
    "        self.optim.step()\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(agent, venv, buf, episodes=100, T=1000, batch_size=32, alpha=0.01, eps=0.):\n",
    "    '''\n",
    "     ag.train(agent, venv, buf, episodes=100, T=1000, batch_size=32, alpha=0.01, eps=0.)\n",
    "\n",
    "     Trains the agent on the environ\n",
    "    '''\n",
    "    # First, let's populate a replay buffer\n",
    "    #buf = ReplayBuffer(size=T*batch_size)\n",
    "    for e in tqdm(range(episodes)):\n",
    "        s = venv.reset()\n",
    "        for k in range(T):\n",
    "            a = agent.choose_action(s, eps=eps)\n",
    "            sp, r, done, _ = venv.step(a)\n",
    "            buf.store(s, a, r, sp, done)\n",
    "            s = sp\n",
    "\n",
    "            batch = buf.sample(batch_size)\n",
    "            agent.update(batch[0], batch[1], batch[2], batch[3], batch[4])\n",
    "            if all(done):\n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simulate(agent, trials=1, eps=0., eps_decay=0.995):\n",
    "    time.sleep(1)\n",
    "    avg_k = 0.\n",
    "    for trial in range(trials):\n",
    "        s = agent.venv.envs[0].reset()\n",
    "        for k in range(2000):\n",
    "            agent.venv.envs[0].render()\n",
    "            a = agent.choose_action([s], eps=eps*eps_decay**k)\n",
    "\n",
    "            sp, r, done, info = agent.venv.envs[0].step(a[0])\n",
    "            s = sp\n",
    "            if done:\n",
    "                print('Done after '+str(k))\n",
    "                avg_k += k\n",
    "                break\n",
    "                \n",
    "    print('Average reward: '+str(avg_k/trials))\n",
    "    time.sleep(1)\n",
    "    agent.venv.envs[0].close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "def episode(agent, env, s=None, max_iters=100, eps=0.):\n",
    "    done = False\n",
    "    n_iters = 0\n",
    "    if s is None:\n",
    "        s = env.reset()\n",
    "    else:\n",
    "        env.coords = np.array(s)\n",
    "    traj = []\n",
    "    while not done and n_iters<max_iters:\n",
    "        a = agent.choose_action([s], eps=eps)\n",
    "        sp, r, done, _ = env.step(a[0])\n",
    "        #print(s, a, r, sp, done)\n",
    "        traj.append(dict(a=a[0], s=s, sp=sp, r=r, d=done))\n",
    "        s = sp\n",
    "        n_iters += 1\n",
    "    print(n_iters, done)\n",
    "    return traj"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train and Test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## CartPole"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 480,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "venv = Vector_env((lambda : gym.make('CartPole-v0')), n=32)\n",
    "ag = Agent(venv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 484,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10000/10000 [00:16<00:00, 611.04it/s]\n"
     ]
    }
   ],
   "source": [
    "buf = train(ag, venv, T=10000, alpha=0.001, eps=0., batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 485,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done after 199\n",
      "Done after 199\n",
      "Done after 199\n",
      "Average reward: 199.0\n"
     ]
    }
   ],
   "source": [
    "simulate(ag, trials=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MoonLander"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [],
   "source": [
    "venv = Vector_env((lambda : gym.make('LunarLander-v2')), n=1)\n",
    "ag = Agent(venv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [],
   "source": [
    "buf = ReplayBuffer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50/50 [00:08<00:00,  5.59it/s]\n",
      "100%|██████████| 50/50 [00:08<00:00,  5.84it/s]\n",
      "100%|██████████| 50/50 [00:11<00:00,  4.32it/s]\n",
      "100%|██████████| 50/50 [00:09<00:00,  5.43it/s]\n",
      "100%|██████████| 50/50 [00:08<00:00,  5.59it/s]\n",
      "100%|██████████| 50/50 [00:11<00:00,  4.21it/s]\n",
      "100%|██████████| 50/50 [00:11<00:00,  4.31it/s]\n",
      "100%|██████████| 50/50 [00:19<00:00,  2.61it/s]\n",
      "100%|██████████| 50/50 [00:21<00:00,  2.35it/s]\n",
      "100%|██████████| 50/50 [00:26<00:00,  1.92it/s]\n"
     ]
    }
   ],
   "source": [
    "venv.set_max_episode_steps(1000)\n",
    "epochs = 10\n",
    "for epoch in range(epochs):\n",
    "    eps = np.exp(-epoch/10)\n",
    "    train(ag, venv, buf, episodes=50, alpha=0.001, eps=eps, batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done after 506\n",
      "Done after 999\n",
      "Done after 999\n",
      "Average reward: 834.6666666666666\n"
     ]
    }
   ],
   "source": [
    "venv.set_max_episode_steps(1000)\n",
    "simulate(ag, trials=3, eps=0.2, eps_decay=0.995)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.022688763303990733"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "0.5*0.995**617."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GridWorld"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 443,
   "metadata": {},
   "outputs": [],
   "source": [
    "venv = Vector_env((lambda : GridWorld(8, goal=[([4,1],10.),([4,6],10.)])), n=32)\n",
    "ag = Agent(venv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 445,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:00<00:00, 451.05it/s]\n"
     ]
    }
   ],
   "source": [
    "buf = train(ag, venv, T=100, alpha=0.001, eps=0.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 446,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:01<00:00, 533.22it/s]\n"
     ]
    }
   ],
   "source": [
    "buf = train(ag, venv, T=1000, alpha=0.001, eps=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 464,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 True\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeUAAAHWCAYAAABJ3pFhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAc/klEQVR4nO3df3Bd9Xnn8c9j2cbCwrh1FMe/UgLFN4WGYGxoqUtGAhogCanLwBS2MEu2u9pNC01oQwLtsCybacNMsiHZusPUJYQUUlLhGhpIgiFx7towgEE22NhGDjbgH3JsYyLsC/Iv6dk/dJ1a/inpHHKec877NeMB6V597/exfP32PfeeK3N3AQCA7I3IegMAAKAfUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgkgUZTO7ysxWmVmfmc1Ka1MAAJRR0kfKL0u6QtLiFPYCAECpjUzyxe6+RpLMLJ3dAABQYjynDABAEMd9pGxmP5b0gSNc9Dfu/u+DvSEza5PUJkljxoyZ+cEPfnDQm4yqr69PI0bk+981RZhBKsYcRZhBYo5IijCDVIw51q5d+6a7Nx/veseNsrtfnMaG3H2epHmSVKlUvLOzM41lM1WtVtXS0pL1NhIpwgxSMeYowgwSc0RShBmkYsxhZm8M5nr5/qcHAAAFkvSUqD8ys02Szpf0AzNbmM62AAAon6Svvn5Y0sMp7QUAgFLj8DUAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIFKJspldamadZvaqmd2SxpoAAJRN4iibWYOkf5B0maQzJF1jZmckXRcAgLJJ45HyeZJedff17r5X0vck/WEK6wIAUCrm7skWMLtS0qXu/l/rH18n6Xfc/YZDrtcmqU2SmpubZ7a3tye63QhqtZqampqy3kYiRZhBKsYcRZhBYo5IijCDVIw5WltbO9x91vGuNzKF27IjfO6w0rv7PEnzJKlSqXhLS0sKN52tarWqvM9RhBmkYsxRhBkk5oikCDNIxZljMNI4fL1J0rSDPp4qqSuFdQEAKJU0ovy8pNPN7ENmNlrS1ZK+n8K6AACUSuLD1+6+38xukLRQUoOke919VeKdAQBQMmk8pyx3/6GkH6axFgAAZcU7egEAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABJFKlM3sXjPbZmYvp7EeAABllNYj5fskXZrSWgAAlFIqUXb3xZLeSmMtAADKiueUAQAIwtw9nYXMTpH0mLv/9lEub5PUJknNzc0z29vbU7ndLNVqNTU1NWW9jUSKMINUjDmKMIPEHJEUYQapGHO0trZ2uPus413vVxblg1UqFe/s7EzldrNUrVbV0tKS9TYSKcIMUjHmKMIMEnNEUoQZpGLMYWaDijKHrwEACCKtU6IelPSMpIqZbTKzP01jXQAAymRkGou4+zVprAMAQJlx+BoAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIIjEUTazaWb2UzNbY2arzOxzaWwMAICyGZnCGvsl/ZW7LzOzkyR1mNmT7r46hbUBACiNxI+U3X2Luy+r//8uSWskTUm6LgAAZZPqc8pmdoqkGZKeS3NdAADKwNw9nYXMmiT9P0l/6+4LjnB5m6Q2SWpubp7Z3t6eyu1mqVarqampKettJFKEGaRizFGEGSTmiKQIM0jFmKO1tbXD3Wcd73qpRNnMRkl6TNJCd//68a5fqVS8s7Mz8e1mrVqtqqWlJettJFKEGaRizFGEGSTmiKQIM0jFmMPMBhXlNF59bZK+JWnNYIIMAACOLI3nlGdLuk7ShWb2Yv3XJ1JYFwCAUkl8SpS7PyXJUtgLAAClxjt6AQAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEkTjKZjbGzJaa2UtmtsrM7khjYwAAlM3IFNbYI+lCd6+Z2ShJT5nZj9z92RTWBgCgNBJH2d1dUq3+4aj6L0+6LgAAZZPKc8pm1mBmL0raJulJd38ujXUBACgT63+gm9JiZuMlPSzpRnd/+ZDL2iS1SVJzc/PM9vb21G43K7VaTU1NTVlvI5EizCAVY44izCAxRyRFmEEqxhytra0d7j7reNdLNcqSZGa3S3rH3b92tOtUKhXv7OxM9XazUK1W1dLSkvU2EinCDFIx5ijCDBJzRFKEGaRizGFmg4pyGq++bq4/QpaZNUq6WNIrSdcFAKBs0nj19SRJ3zGzBvVHvt3dH0thXQAASiWNV1+vkDQjhb0AAFBqvKMXAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRLmEenrWae3aP9OSJeMkXaglS8Zp7do/U0/Puqy3BiAh7t/5llqUzazBzJab2WNprYn07djxIz3//Fnq6rpHvb27JLl6e3epq+sePf/8Wdqx40dZbxHAMHH/zr80Hyl/TtKaFNdDynp61mnVqivV1/euuh86TdPnTNDHWk3T50xQ90Onqa/vXa1adWWu/kX9yPLNmn3nIl3/+DuafeciPbJ8c9ZbGrIizIDsHXz/lvYdcum+XN6/yyiVKJvZVEmflHRPGuvhvbFx4/9RX98+dT/0YX183npNfnuHRsg1+e0d+vi89ep+6MPq69unjRvvynqrg/LI8s26dcFKbe7ukSRt7u7RrQtW5ipqRZgBMRy4fx9Lnu7fZTUypXW+IemLkk5KaT28B7ZufUDSPp333e06cf/eAZeduH+vzn1ghxZf9n41bL1fG/bfpn9asv6wNe7647M1eXyjHn2pSw88+8Zhl9997Uz9+tjReuiFjZrfsemwy+/7zHlqHN2g+595XY+t2HLY5f/638+XJM1bvE4/WbNtwGVjRjXoO//lPEnS//3JzzR30ava29s34Do9+3r11YWd6ty6S8ve+MWAyyadPEbfuHqGJOmOR1dpddfOAZef2jxWX7niLEnSrQtWaP32dwZcfsbkcbr98jMlSZ//3nJteXv3gMvP+Y1f05cu/bAk6X/c36FfvDvw93j2b75Pf3HR6ZKk/3zvUu3e16vlG7qPOsOcGVMO+/0BjubA/fvY9mnr1vs1ffrcX8WWMAyJo2xmn5K0zd07zKzlGNdrk9QmSc3NzapWq0lvOnO1Wi1nc9QkSR94+60jXjpp55uSJqu3d5dWvrxS3d2H38GfeeYZTWgcodVb9h/x8qefflonjTa9smmfurv3H3b54iWLdUKDae2GI19+4Pdz3WuHXz66wX55+Wuv7T0sZgds7u7Rhjc2qLu7d8DnR+z+j6/ftGmPuncO/PqufTtVrfb/3nR17VH3OwMv39S3U9XqdknS1q271b3bB1y+QbtUrf5ckrT9zd2q7R14+Wuv7VK12v8I+K23dmtvrx9zhnz92foP+btfHFn+5qgN6lq9vbtyNlcevxfDZ+5+/GsdawGzr0i6TtJ+SWMkjZO0wN2vPdrXVCoV7+zsTHS7EVSrVbW0tGS9jUFbsmScent3afqcCZr89o7DLu86eYLWPrJDDQ3jdMEFb2eww6GZfeeiXx72PdiU8Y16+pYLM9jR0BVhhkPl7X5xNHmb48D9+3jycv8+WN6+F0diZh3uPut410v8nLK73+ruU939FElXS1p0rCAjOxMnXitplJb+SbPeHTl6wGXvjhytpX/SLGmUJk68LpP9DdXNl1TUOKphwOcaRzXo5ksqGe1o6IowA2I4cP8+tvzcv8uK85RLZNq0v9KIEaM0/qpX9ETbqeo6eYL6ZOo6eYKeaDtV4696RSNGjNK0aTdlvdVBmTNjir5yxUc0ZXyjpP5Hl1+54iO5ei72wAyjG/rvinmcATEcuH8fS57u32WV1gu9JEnuXpVUTXNNpKex8TSdeeZ8rVp1pcZftU5rr9qntZKkHRqvnRox4kSdeeZ8NTaelvFOB2/OjCmaM2NKrg9vzZkxRQ8u3aDu7m4t/FI+D1kjewffv/tfhX3waz5GacSIUbm7f5cRj5RLZsKEy3TuuSs0eXKbGhrGSTI1NIzT5MltOvfcFZow4bKstwhgmLh/51+qj5SRD42Np2n69LmaPn2uqtWqLrigJestAUgJ9+9845EyAABBEGUggE+dNUnnTeLAFVB2RBkI4LrzT9FFHzze6SwAio4oAwH07O3Vnt5kb+QDIP+IMhDA9d9eqq+/sPv4VwRQaEQZAIAgiDIAAEEQZQAAgiDKAAAEQZSBAK6cOVW/P4XzlIGyI8pAAFfNmqYLpnKeMlB2RBkI4K139mrXXs5TBsqOKAMBfPaBDs1dznnKQNkRZQAAgiDKAAAEQZQBAAiCKAMAEAQnRgIBXPu7v6HVq9/JehsAMkaUgQAu/+hknfSLtVlvA0DGOHwNBNDV3aMdPX1ZbwNAxogyEMBN//qi5q3Yk/U2AGSMKAMAEARRBgAgCKIMAEAQRBkAgCCIMhDAf7vgVF36IX50I1B2nKcMBHDxGRM1ctuarLcBIGM8UgYCWLe9pi01zlMGyo4oAwH89YKVum8V5ykDZUeUAQAIgigDABAEUQYAIAiiDABAEEQZCODGC0/Xp08bnfU2AGSM85SBAH7/9Pdp/+aGrLcBIGM8UgYCWNX1tt7Y2Zv1NgBkjCgDAfzvR1frX9bszXobADKWyuFrM3td0i5JvZL2u/usNNYFAKBM0nyk3OruZxNkYGgeWb5Zyzd0q/MXfZp95yI9snxz1lsCkBEOXwMZemT5Zt26YKX29va/7/Xm7h7dumAlYQZKKq0ou6QnzKzDzNpSWhMovK8u7FTPvoEv8OrZ16uvLuzMaEcAsmTunnwRs8nu3mVm75f0pKQb3X3xIddpk9QmSc3NzTPb29sT327WarWampqast5GIkWYQcrvHNc//s5RL7vv0rG/wp2kJ6/fi0MVYY4izCAVY47W1taOwTy9m0qUByxo9r8k1dz9a0e7TqVS8c7O/D8SqFaramlpyXobiRRhBim/c8y+c5E2d/cc9vkp4xv19C0XZrCj5PL6vThUEeYowgxSMeYws0FFOfHhazMba2YnHfh/SR+X9HLSdYEyuPmSihpHDXzTkMZRDbr5kkpGOwKQpTROiZoo6WEzO7Dev7j74ymsCxTenBlTJElfnL9Ce3v7NGV8o26+pPLLzwMol8RRdvf1kj6awl6AUpozY4oeXLpB3d3dWvilfB6yBpAOTokCACAIogwAQBBEGQCAIIgyEMD/vPwM/aff4ucpA2XHz1MGAjhz8snaPo6fpwyUHY+UgQCe+tmbWvUmP08ZKDuiDATw94t+pu+v4+cpA2VHlAEACIIoAwAQBFEGACAIogwAQBBEGQjg7674iK4/84SstwEgY0QZCOC05iZNauLuCJQdfwsAAfx49VYt37Y/620AyBhRBgL4pyXr9fhr+7LeBoCMEWUAAIIgygAABEGUAQAIgigDABAEUQYCuOuPz1bbWZynDJQdUQYCmDy+URMauTsCZcffAkAAj77Upee2cJ4yUHZEGQjggWff0KINnKcMlB1RBgAgCKIMAEAQRBkAgCCIMgAAQRBlIIC7r52pG2aMyXobADJGlIEAfn3saJ002rLeBoCMEWUggIde2KglmzglCii7kVlvAIA0v2OTurt58xCg7HikDABAEEQZAIAgiDIAAEEQZQAAgiDKQAD3feY8/eUszlMGyo4oAwE0jm7QCQ2cpwyUHVEGArj/mdf1E350I1B6nKcMBPDYii2cpwwgnUfKZjbezOab2StmtsbMzk9jXQAAyiStw9fflPS4u39Y0kclrUlpXaDwbnp0rhZsmaMnej6pMXdM1E2Pzs16SwAykvjwtZmNk/QxSddLkrvvlbQ36bpAGdz06Fx9s+MLctsjSdqjbfpmxxckSXddfkOWWwOQgTQeKZ8qabukb5vZcjO7x8zGprAuUHh3L/vyL4N8gNse3b3syxntCECWzN2TLWA2S9Kzkma7+3Nm9k1JO939tkOu1yapTZKam5tntre3J7rdCGq1mpqamrLeRiJFmEHK7xyt1QslO8J90E0/bVn0q99QCvL6vThUEeYowgxSMeZobW3tcPdZx7teGlH+gKRn3f2U+scXSLrF3T95tK+pVCre2dmZ6HYjqFaramlpyXobiRRhBim/c4y5Y6L2aNthnz9B79fu27dmsKPk8vq9OFQR5ijCDFIx5jCzQUU58eFrd/+5pI1mVql/6iJJq5OuC5TBZ8+5TeYnDPic+Qn67Dm3HeUrABRZWucp3yjpu2Y2WtJ6SZ9JaV2g0A68mOvuZV/WHt+uE6xZn515Gy/yAkoqlSi7+4uSjvuwHMDh7rr8Bt11+Q2FOEQHIBneZhMAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEImjbGYVM3vxoF87zezzaWwOAIAyGZl0AXfvlHS2JJlZg6TNkh5Oui4AAGWT9uHriyStc/c3Ul4XAIDCM3dPbzGzeyUtc/e5R7isTVKbJDU3N89sb29P7XazUqvV1NTUlPU2EinCDFIx5ijCDBJzRFKEGaRizNHa2trh7rOOd73UomxmoyV1STrT3bce67qVSsU7OztTud0sVatVtbS0ZL2NRIowg1SMOYowg8QckRRhBqkYc5jZoKKc5uHry9T/KPmYQQYAAEeWZpSvkfRgiusBAFAqqUTZzE6U9AeSFqSxHgAAZZT4lChJcvd3JU1IYy0AAMqKd/QCACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAiCKAMAEARRBgAgCKIMAEAQRBkAgCCIMgAAQRBlAACCIMoAAARBlAEACIIoAwAQBFEGACAIogwAQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIAiiDABAEEQZAIAgiDIAAEEQZQAAgiDKAAAEQZQBAAgilSib2U1mtsrMXjazB81sTBrrAgBQJomjbGZTJP2FpFnu/tuSGiRdnXRdAADKJq3D1yMlNZrZSEknSupKaV0AAEojcZTdfbOkr0naIGmLpLfd/Ymk6wIAUDYjky5gZr8m6Q8lfUhSt6SHzOxad3/gkOu1SWqrf7jHzF5OetsBvE/Sm1lvIqEizCAVY44izCAxRyRFmEEqxhyVwVwpcZQlXSzpNXffLklmtkDS70kaEGV3nydpXv06L7j7rBRuO1NFmKMIM0jFmKMIM0jMEUkRZpCKMYeZvTCY66XxnPIGSb9rZieamUm6SNKaFNYFAKBU0nhO+TlJ8yUtk7Syvua8pOsCAFA2aRy+lrvfLun2IXxJUaJdhDmKMINUjDmKMIPEHJEUYQapGHMMagZz9/d6IwAAYBB4m00AAILILMpmdlX9rTn7zCxXr6ozs0vNrNPMXjWzW7Lez3CY2b1mti3Pp6aZ2TQz+6mZran/Wfpc1nsaDjMbY2ZLzeyl+hx3ZL2n4TKzBjNbbmaPZb2X4TKz181spZm9ONhXzEZkZuPNbL6ZvVK/j5yf9Z6Gwswq9e/BgV87zezzWe9rOIbyVtSZHb42s9+S1CfpHyV9wd1z8YffzBokrZX0B5I2SXpe0jXuvjrTjQ2RmX1MUk3SP9ffHjV3zGySpEnuvszMTpLUIWlODr8XJmmsu9fMbJSkpyR9zt2fzXhrQ2ZmfylplqRx7v6prPczHGb2uvrfNjjX58Wa2XckLXH3e8xstKQT3b07630NR/3v3c2Sfsfd38h6P0NRfyvqpySd4e49ZtYu6Yfuft+Rrp/ZI2V3X+PunVndfgLnSXrV3de7+15J31P/m6fkirsvlvRW1vtIwt23uPuy+v/vUv+peFOy3dXQeb9a/cNR9V+5e7GHmU2V9ElJ92S9l7Izs3GSPibpW5Lk7nvzGuS6iySty1uQDzLot6LmOeWhmyJp40Efb1IOQ1A0ZnaKpBmSnst2J8NTP+z7oqRtkp6sn2qYN9+Q9EX1HwHLM5f0hJl11N+JMI9OlbRd0rfrTyfcY2Zjs95UAldLejDrTQzHUN+K+j2Nspn9uH4M/dBfuXtkeRA7wudy96imSMysSdK/Sfq8u+/Mej/D4e697n62pKmSzjOzXD2lYGafkrTN3Tuy3ksKZrv7OZIuk/Tn9ad68makpHMk3e3uMyS9Iymvr38ZLenTkh7Kei/DcchbUU+WNNbMrj3a9VM5T/lo3P3i93L9jGySNO2gj6eKn4qVmfpzsP8m6bvuviDr/STl7t1mVpV0qaQ8vQhvtqRPm9knJI2RNM7MHnD3o/7lE5W7d9X/u83MHlb/U1aLs93VkG2StOmgIy7zldMoq/8fR8vcfWvWGxmmQb0V9QEcvh665yWdbmYfqv8L7mpJ3894T6VUf4HUtyStcfevZ72f4TKzZjMbX///RvXfiV/JdldD4+63uvtUdz9F/feJRXkMspmNrb9oUPXDvR9Xvv5xJEly959L2mhmB34IwkWScvUCyINco5weuq4b0ltRZ3lK1B+Z2SZJ50v6gZktzGovQ+Hu+yXdIGmh+n9j2919Vba7Gjoze1DSM5IqZrbJzP406z0Nw2xJ10m68KDTJj6R9aaGYZKkn5rZCvX/o+9Jd8/tKUU5N1HSU2b2kqSlkn7g7o9nvKfhulHSd+t/rs6W9HcZ72fIzOxE9Z/pktujYEN9K2re0QsAgCA4fA0AQBBEGQCAIIgyAABBEGUAAIIgygAABEGUAQAIgigDABAEUQYAIIj/DxRSd5MpEhjCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#traj = episode(ag, env, s=[1,3], eps=0.)\n",
    "traj = episode(ag, ag.venv.envs[0], s=None, eps=0.)\n",
    "ag.venv.envs[0].draw(traj=traj);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5 7]\n"
     ]
    }
   ],
   "source": [
    "s = ag.venv.envs[0].reset()\n",
    "print(s)\n",
    "#env.render()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = np.array([[5,6]])\n",
    "Qa = ag.Q(torch.as_tensor(s).float())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[6.3182, 5.3722, 6.7821, 7.1567]], grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 221,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Qa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}