Noisy_DQN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Noisy-DQN.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "LNK8PI5zfca9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "9317c5d7-a4a7-4b80-9d5a-fd59028ec7a5"
      },
      "source": [
        "import numpy as np\n",
        "import random\n",
        "from collections import namedtuple, deque\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import gym\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "BUFFER_SIZE = int(1e5)  # replay buffer size\n",
        "BATCH_SIZE = 64         # minibatch size\n",
        "GAMMA = 0.99            # discount factor\n",
        "TAU = 1e-3              # for soft update of target parameters\n",
        "LR = 1e-3               # learning rate \n",
        "UPDATE_EVERY = 1        # how often to update the network\n",
        "\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "print(\"using: \", device)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "using:  cuda:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9BcNpOLmfuUn",
        "colab_type": "text"
      },
      "source": [
        "# **Regular Dueling DQN**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q4lf6GSpfkSY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Dueling_QNetwork(nn.Module):\n",
        "    \"\"\"Actor (Policy) Model.\"\"\"\n",
        "\n",
        "    def __init__(self, state_size, action_size, seed, fc1_units=64, fc2_units=64):\n",
        "        \"\"\"Initialize parameters and build model.\n",
        "        Params\n",
        "        ======\n",
        "            state_size (int): Dimension of each state\n",
        "            action_size (int): Dimension of each action\n",
        "            seed (int): Random seed\n",
        "            fc1_units (int): Number of nodes in first hidden layer\n",
        "            fc2_units (int): Number of nodes in second hidden layer\n",
        "        \"\"\"\n",
        "        super(Dueling_QNetwork, self).__init__()\n",
        "        self.action_size = action_size\n",
        "        self.seed = torch.manual_seed(seed)\n",
        "        self.network = nn.Sequential(nn.Linear(state_size,fc1_units),\n",
        "                                     nn.ReLU(),\n",
        "                                     nn.Linear(fc1_units,fc2_units),\n",
        "                                     nn.ReLU())\n",
        "        self.advantage = nn.Sequential(nn.Linear(fc2_units,action_size))\n",
        "        self.value = nn.Sequential(nn.Linear(fc2_units,1))\n",
        "\n",
        "    def forward(self, state):\n",
        "        x = self.network(state)\n",
        "        value = self.value(x)\n",
        "        value = value.expand(x.size(0), self.action_size)\n",
        "        advantage = self.advantage(x)\n",
        "        Q = value + advantage - advantage.mean()\n",
        "        return Q\n",
        "\n",
        "class Agent():\n",
        "    \"\"\"Interacts with and learns from the environment.\"\"\"\n",
        "\n",
        "    def __init__(self, state_size, action_size, seed):\n",
        "        \"\"\"Initialize an Agent object.\n",
        "        \n",
        "        Params\n",
        "        ======\n",
        "            state_size (int): dimension of each state\n",
        "            action_size (int): dimension of each action\n",
        "            seed (int): random seed\n",
        "        \"\"\"\n",
        "        self.state_size = state_size\n",
        "        self.action_size = action_size\n",
        "        self.seed = random.seed(seed)\n",
        "        \n",
        "\t    # Q-Network\n",
        "        \n",
        "        self.qnetwork_local = Dueling_QNetwork(state_size, action_size, seed).to(device)\n",
        "        self.qnetwork_target = Dueling_QNetwork(state_size, action_size, seed).to(device)\n",
        "        #if name == \"DoubleDQN\":\n",
        "        #    self.qnetwork_local = DDQN.QNetwork(state_size, action_size, seed).to(device)\n",
        "        #    self.qnetwork_target = DDQN.QNetwork(state_size, action_size, seed).to(device)\n",
        "\n",
        "\n",
        "        self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=LR)\n",
        "\n",
        "        # Replay memory\n",
        "        self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed)\n",
        "        # Initialize time step (for updating every UPDATE_EVERY steps)\n",
        "        self.t_step = 0\n",
        "    \n",
        "    def step(self, state, action, reward, next_state, done):\n",
        "        # Save experience in replay memory\n",
        "        self.memory.add(state, action, reward, next_state, done)\n",
        "        \n",
        "        # Learn every UPDATE_EVERY time steps.\n",
        "        self.t_step = (self.t_step + 1) % UPDATE_EVERY\n",
        "        if self.t_step == 0:\n",
        "            # If enough samples are available in memory, get random subset and learn\n",
        "            if len(self.memory) > BATCH_SIZE:\n",
        "                experiences = self.memory.sample()\n",
        "                self.learn(experiences, GAMMA)\n",
        "\n",
        "    def act(self, state, eps=0.):\n",
        "        \"\"\"Returns actions for given state as per current policy.\n",
        "        \n",
        "        Params\n",
        "        ======\n",
        "            state (array_like): current state\n",
        "            eps (float): epsilon, for epsilon-greedy action selection\n",
        "        \"\"\"\n",
        "        state = torch.from_numpy(state).float().unsqueeze(0).to(device)\n",
        "        self.qnetwork_local.eval()\n",
        "        with torch.no_grad():\n",
        "            action_values = self.qnetwork_local(state)\n",
        "        self.qnetwork_local.train()\n",
        "\n",
        "        # Epsilon-greedy action selection\n",
        "        if random.random() > eps:\n",
        "            return np.argmax(action_values.cpu().data.numpy())\n",
        "        else:\n",
        "            return random.choice(np.arange(self.action_size))\n",
        "\n",
        "    def learn(self, experiences, gamma):\n",
        "        \"\"\"Update value parameters using given batch of experience tuples.\n",
        "        Params\n",
        "        ======\n",
        "            experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples \n",
        "            gamma (float): discount factor\n",
        "        \"\"\"\n",
        "        states, actions, rewards, next_states, dones = experiences\n",
        "\n",
        "        # Get max predicted Q values (for next states) from target model\n",
        "        Q_targets_next = self.qnetwork_target(next_states).detach().max(1)[0].unsqueeze(1)\n",
        "        # Compute Q targets for current states \n",
        "        Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))\n",
        "\n",
        "        # Get expected Q values from local model\n",
        "        Q_expected = self.qnetwork_local(states).gather(1, actions)\n",
        "\n",
        "        # Compute loss\n",
        "        loss = F.mse_loss(Q_expected, Q_targets)\n",
        "        # Minimize the loss\n",
        "        self.optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        self.optimizer.step()\n",
        "\n",
        "        # ------------------- update target network ------------------- #\n",
        "        self.soft_update(self.qnetwork_local, self.qnetwork_target, TAU)                     \n",
        "\n",
        "    def soft_update(self, local_model, target_model, tau):\n",
        "        \"\"\"Soft update model parameters.\n",
        "        θ_target = τ*θ_local + (1 - τ)*θ_target\n",
        "        Params\n",
        "        ======\n",
        "            local_model (PyTorch model): weights will be copied from\n",
        "            target_model (PyTorch model): weights will be copied to\n",
        "            tau (float): interpolation parameter \n",
        "        \"\"\"\n",
        "        for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):\n",
        "            target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)\n",
        "\n",
        "\n",
        "class ReplayBuffer:\n",
        "    \"\"\"Fixed-size buffer to store experience tuples.\"\"\"\n",
        "\n",
        "    def __init__(self, action_size, buffer_size, batch_size, seed):\n",
        "        \"\"\"Initialize a ReplayBuffer object.\n",
        "        Params\n",
        "        ======\n",
        "            action_size (int): dimension of each action\n",
        "            buffer_size (int): maximum size of buffer\n",
        "            batch_size (int): size of each training batch\n",
        "            seed (int): random seed\n",
        "        \"\"\"\n",
        "        self.action_size = action_size\n",
        "        self.memory = deque(maxlen=buffer_size)  \n",
        "        self.batch_size = batch_size\n",
        "        self.experience = namedtuple(\"Experience\", field_names=[\"state\", \"action\", \"reward\", \"next_state\", \"done\"])\n",
        "        self.seed = random.seed(seed)\n",
        "    \n",
        "    def add(self, state, action, reward, next_state, done):\n",
        "        \"\"\"Add a new experience to memory.\"\"\"\n",
        "        e = self.experience(state, action, reward, next_state, done)\n",
        "        self.memory.append(e)\n",
        "    \n",
        "    def sample(self):\n",
        "        \"\"\"Randomly sample a batch of experiences from memory.\"\"\"\n",
        "        experiences = random.sample(self.memory, k=self.batch_size)\n",
        "\n",
        "        states = torch.from_numpy(np.vstack([e.state for e in experiences if e is not None])).float().to(device)\n",
        "        actions = torch.from_numpy(np.vstack([e.action for e in experiences if e is not None])).long().to(device)\n",
        "        rewards = torch.from_numpy(np.vstack([e.reward for e in experiences if e is not None])).float().to(device)\n",
        "        next_states = torch.from_numpy(np.vstack([e.next_state for e in experiences if e is not None])).float().to(device)\n",
        "        dones = torch.from_numpy(np.vstack([e.done for e in experiences if e is not None]).astype(np.uint8)).float().to(device)\n",
        "  \n",
        "        return (states, actions, rewards, next_states, dones)\n",
        "\n",
        "    def __len__(self):\n",
        "        \"\"\"Return the current size of internal memory.\"\"\"\n",
        "        return len(self.memory)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YH4ytAkOfkQC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 368
        },
        "outputId": "048d309d-ca11-4bbd-d7cc-41bb115081b2"
      },
      "source": [
        "def dqn(n_episodes=1000, max_t=1000, eps_start=1.0, eps_end=0.01, eps_decay=0.99):\n",
        "    \"\"\"Deep Q-Learning.\n",
        "    \n",
        "    Params\n",
        "    ======\n",
        "        n_episodes (int): maximum number of training episodes\n",
        "        max_t (int): maximum number of timesteps per episode\n",
        "        eps_start (float): starting value of epsilon, for epsilon-greedy action selection\n",
        "        eps_end (float): minimum value of epsilon\n",
        "        eps_decay (float): multiplicative factor (per episode) for decreasing epsilon\n",
        "    \"\"\"\n",
        "    scores = []                        # list containing scores from each episode\n",
        "    scores_window = deque(maxlen=100)  # last 100 scores\n",
        "    eps = eps_start                    # initialize epsilon\n",
        "    for i_episode in range(1, n_episodes+1):\n",
        "        state = env.reset()\n",
        "        score = 0\n",
        "        for t in range(max_t):\n",
        "            action = agent.act(state, eps)\n",
        "            next_state, reward, done, _ = env.step(action)\n",
        "            agent.step(state, action, reward, next_state, done)\n",
        "            state = next_state\n",
        "            score += reward\n",
        "            if done:\n",
        "                break \n",
        "        scores_window.append(score)       # save most recent score\n",
        "        scores.append(score)              # save most recent score\n",
        "        eps = max(eps_end, eps_decay*eps) # decrease epsilon\n",
        "        print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)), end=\"\")\n",
        "        if i_episode % 100 == 0:\n",
        "            print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)))\n",
        "        if np.mean(scores_window)>=200.0:\n",
        "            print('\\nEnvironment solved in {:d} episodes!\\tAverage Score: {:.2f}'.format(i_episode-100, np.mean(scores_window)))\n",
        "            torch.save(agent.qnetwork_local.state_dict(), 'checkpoint.pth')\n",
        "            break\n",
        "    return scores\n",
        "\n",
        "\n",
        "env = gym.make(\"CartPole-v0\")\n",
        "action_size = env.action_space.n\n",
        "state_size = env.observation_space.shape[0]\n",
        "\n",
        "agent = Agent(state_size=state_size, action_size=action_size, seed=0)\n",
        "scores = dqn(n_episodes = 500)\n",
        "\n",
        "# plot the scores\n",
        "fig = plt.figure()\n",
        "ax = fig.add_subplot(111)\n",
        "plt.plot(np.arange(len(scores)), scores)\n",
        "plt.ylabel('Score')\n",
        "plt.xlabel('Episode #')\n",
        "plt.show()\n"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Episode 100\tAverage Score: 16.36\n",
            "Episode 200\tAverage Score: 26.40\n",
            "Episode 300\tAverage Score: 129.62\n",
            "Episode 400\tAverage Score: 164.43\n",
            "Episode 500\tAverage Score: 171.21\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsnXecHMWZ939Pz8wm7UqrsFplrYQk\nFAgClpxzssH2YTC2MdjY2H7BOJ1tjO8wjsf5HO547RcDNjbYhsMGR5LJOYokhAjKKGuVdqVNk+r9\no7t6qquranp2Z3ZnZ+r7+eizM9Wpulf7PP3EIsYYLBaLxWKRcYZ7AhaLxWIpT6yCsFgsFosSqyAs\nFovFosQqCIvFYrEosQrCYrFYLEqsgrBYLBaLEqsgLBaLxaLEKgiLxWKxKLEKwmKxWCxK4sM9gcEw\nYcIE1tbWNtzTsFgslhHFyy+/vJ0x1pJvvxGtINra2rBkyZLhnobFYrGMKIhoXZT9rIvJYrFYLEqs\ngrBYLBaLEqsgLBaLxaLEKgiLxWKxKLEKwmKxWCxKSqYgiGg6ET1GRMuJ6E0i+qI3Po6IHiKiFd7P\nsd44EdH1RLSSiJYS0cGlmpvFYrFY8lNKCyIN4KuMsYUAjgBwOREtBHAVgEcYY3MBPOJ9B4AzAcz1\n/l0G4IYSzs1isVgseShZHQRjbDOAzd7nPUT0FoCpAM4FcIK3260AHgfwDW/8Nuaugfo8ETUT0WTv\nPBbLkPHyul3YvrcfDTUxHDs3V0u0fFMXelMZdPYm8dp7uxFzHHzksOloHV1X8jmt3LYX9y7djA8d\nPBXTxzUo99nZncTzq3fgrP0nF3x+xhjuenkDzl08FUs37MYLa3Zi5vgGHDC1GTPG5673xyXrkc0y\ntLeNxb1Lt+C89ml4ftUOnH3AZNQlYgCAZDqLv766EecdMg1PrujAK+t2wXEI57dPx9Mrt+OcA6fg\n3a17AAD7tDTi1ufWoi+ZAQDsO2k0JjTWIOYQtnb1452tewBpWeQ5rU1wCOjqTQMA4g7h/EOn4y+v\nbkB3fwbHz2vBm5u60JNM44MHTUUqw/C319z5vLBmJzp7Uxhdl8CR+4wHADy8fCsWThmNKc31eOLd\nDuzpS2FNRzc+dsRMjBtVgz++tB4bdvXgtEWTsN/UMQCA9Tt7sKpjLyY01mLDrl6kMlls2NWL1tG1\nWL+zF6cubMXCKaMBAKs69uJvr23C2IYEAKChJobte5M4aEYz+lIZrNi6Fx88eCqeW7UD27r60Z1M\nY+7EJsQcQiqTxXs7e5DNMkwaU4ddPUk01MRRE3Nw/qHTC/49F8qQFMoRURuAgwC8AKBVEPpbALR6\nn6cCWC8ctsEbCygIIroMroWBGTNmlGzOlurlX2541v+89rqz/c9nXf8UAGDGuAa8t7MHAFATd/D5\nE/Yp+ZxufXYtfvf8OqSzWXz1tH2V+3z2d0vw0tpdeOlbp6Clqbag89+/bAu+dtdSrNvRg/uWbcbq\njm5/G38GvckMvn7XUgDA+w6YjHuWbsaTKzrw8rpdeGNjJ649ZxEA4LfPrsEP73sbAHDjk6uwyjvX\nw29txbKNXXhvRw9+/thKAMBNFx2CHz3wjnFuRLnPkq7wOWP/Sfjyna+HxiePqcdzq7bj+kdXoi4R\nwxfueDVwX8l0Fp++bQnmTmzEQ185Hhff8qK/fUJTLc45cAq+frd7z6u2d+MXH3U937c+uxZ/enkD\nDpg2Bk+t2B667vLNnbjxonYAwG3PrsWtz4Xr0iY01mD73iQA4L/++Q7SWc3NaThj/0kYXZco6JhC\nKXmQmogaAdwN4EuMsS5xm2ctFPRUGGM3McbaGWPtLS15K8UtlqKTzmRx3iHTQAT0JtNFOefDy7ei\nL5XRXzOb9X7q/1y40uL7FkJnbwoAsH1vP/b0qe8pJZy3P+1+3tntCrhte/r8bb1Jd9v6XT3IZBne\nd4Br0Wza7e6zuzeZ29e750e/ejzmTGwMXXPa2Hqs+Y+z/X8n7Kv+m9c9u+7+NDr29gMAuvpSoe2b\nO3u9OYW39acySGdyzzsjfE5mskhnskim1c9afIbJjPp3xp8dYP696tir+T0Vk5IqCCJKwFUOf2CM\n/dkb3kpEk73tkwFs88Y3AhBtpmnemMVSVmQZECNCXTyGPo2AKITX1+/Gp29bgu/ds1x/zSy/duGC\npFBIM54VhBj/nFEItmbPlbK7JwUGwPFMgF7PjVQXj/n79qfcG6tNxJCIhcVRjTTmUHB2Hzvc9SLo\nBDVDzuogxZ1t3O0qiMljwm7CdJYFnrf4OZNlyDL1/QNATzKnsJjmdzbY3+Te/hGsIIiIAPwawFuM\nsZ8Km/4O4GLv88UA/iaMf8LLZjoCQKeNP1jKkQxjcBygLuH4Qm8w7Oxx3yQ37OrV7sM8cZI1vGma\nBGExEC+dYXoFMbre9Vzv7k2BMcDxpsOthZUde/19+9LuWG3cQU0sPG9ZaTjSLvVe3KM/gqImxWPZ\n6D3zSYo4UiojK4jctkyWgYH5z0FGtGhKpdR1ll4xKWUM4mgAFwF4g4he88auBnAdgD8S0aUA1gE4\n39t2H4CzAKwE0APgkyWcm8UyYBhjICLUJWJGt1BUuNCPydJP3IcFfyrn5f1UCcJiICqDjMGC4HT2\npsDgPiuHcnN//J0Ofx9uQdRpLIhEPHgzJN1cfU0uMK6CMaaNWwA5t9cklQWRyUoKYGAWREajuwar\nN4bCgihlFtPT0FurJyv2ZwAuL9V8LJZikfXeiusTMfSmMli6YTf2nzomJLyiwoWMQT/4b6Gmt9Fi\nvajqboOxsIJQ+c65376zJwnGXCFARMoJ9osWRFyhICSlIU+trgALQkXHXldBxJ3wtVOZbGDKIQuC\nMYOCyAlvnYtpsHSPZBeTxVKpZBmDQ4TaRAwPLt+Kc37+DH7/wnuDOh8Q9q+LcBljljUswj5mTMdm\nFApCJfz4tk7PxQTSK7/eVAYxh5CIOWoLIk8MQkytVSEqbdUU+FxVijeVDSqAQAzCs0x0CqK7P+hi\nKoVVN+KD1BZLJZLNugqiPuH4gmn5ps4Bn4+7IEwuJlaABTEQn3cU+aUSlkoLwhvr6ku77jiQ1rrq\n7E2h1rMcIgWppV3q8ygIxpgfv1Ft48F/laJLZ7LGGETWYEH0pjI5NxwDRtUM3Fmjy9zaYy0Ii6X8\ncAOv5L+9AgN3cQC5tNRoMQiDgkD+faKgC3IzSUCKP8VjxG0MrstKd2u7elL+c6xVupjyxSDcY7ir\nyjRvWUeJQWiVnE9lgvEL2cWWZdAGqYGcmynLGBpqYtr9TMyf1IQbLzpEuc1aEBbLMKN0oTDmxyA4\nKU2uOyedyWKHl48vw4WUWUHoBZlMqTJhA0FqSWGJb+kpzyTKZpkfg9C5zzp7RAsiShZTcJ+aWCxw\nTROy4tva1WdUvKlMVu9iihCk524mNggF0dJUi9q4+tjuItXgmLAKwmIxoBK2WcbgOEELIml4gwWA\n792zHId8/2Fl5onvYooUgzC5mPK7oQZDwN1iEJCiX9/NYtK7sHb1JI0upkTcHKTmSlVnwZmexLE/\negzbPaXdm8yEMtLSUpprMGDN/H108N91Jsu0Qj4fsotNZKSnuVosI54MY3AksZRlrruizmBB8AwY\nnplz7xtuSU9PMo3G2uCfHRe2TgQLwvTGmnMxGW4oArKe4mm9pjdo8e087SsIeBYEaS2I3T0pNNW5\nz0OVxRQulAtuz6cgALOS2LbHVRB/fnUj7lkaLLtKZbOBZyneP7/HpMFyybmYzL9bE3GFVcUZ0YVy\nFksloHobZ14WU10i9+cjuziOvu5RLLjmAeEY/TW4HzuKBWGsgxhEkNoEF4IqYckFJUNYebgWhKtw\ndLe2O58FIQlIWdHEPcGrC1IDeZ690D5EFvayBSEHqd199NflApx5LsmBoHM7fvLoNnzm2FkDO2kB\nWAVhsRhQu5hcYV4fcDEFBcW2Pf1G94tqLIoFYc5i0qeeDoY+r7eSqlBORVqwLphnbenurTuZQa33\nHCPVQUgKwsmjIMTKc1U2kyl2lJKymJR1IMbjc8/BlMI8EE7YdyIOmNZc1HOqsArCYjGgEshZ740w\n6GIyB0n5WVJp9fkAwOBu9t9eTbK/aC4m6TtvhxEMUgcvEnAxZXKNBZlXCGESkNyCiCuUiHxcyMVE\nZhdTOptLc1U9lx5DoDeVyaXBysfzZ5EyNEZMC5bXQF1MOkzWZjGxMQiLxYDqTdl9K3brIDgmXzSQ\nUwIqgcKvYXYxRQhAl8jFxIO3gSCtQQuJzyyZzhqD1EBO0UaRefI+sXwWhOKtX6TH0EsrnZXrIMKf\nzRZI7ndWZP0QqgcpFdaCsFgM6OSgQxTI21dZBiJctqhcElFcTIVYBwPo9q28Foc32QtUUhuUkFg8\n15/O5lptaFDVP+iQLYpckFot6MXnrXK9mZotmrKYRDeaDj/dlxXmYjKlO/v7DJEFYRWExWJgW1cf\n2q66F4++vTUw7lDQJZLMZHHeDc/iSmFBGhHmv3GGpbfvYjL80UeKQUj7DhT58HQmfO2M4c05YEFk\nssZCOQAFpYDKiiafBZFhTLCswttN6zAkpToIUcGYLKjcubM4+/qn8MzKHYgR4f4vHovveIsqmYik\nIIptkmiwCsJiMfDa+t0AgN9JK4I5DiEmBA2S6SyWrNuFv7++SXkePwahUBBcSMUMKY1+QZex3TcP\nUmt3yQsDCykYP23VEIMQke/RlOYKALWJQiyI4HdfQWhcfNksG7DidFttuJ/jDgUUTJQFflIZhjc3\nuWukEQELJo/GflNH5z0uEUH4Fzumob3OkFzFYhmh+MJb+oN0iAJv/HzhGS3cxaQQLNkixSCKZUHI\nU+SpoMEsJv3xstvFlOYKRBOInJCLicwWRDow5wIVhB9kd3//+eIZ/py8+xEVJZ+3qmusTNyUrcD3\nsQrCYhl+eCZKKL2S1FlHuspXkwXBZc1gW20MqlmfcGnZV59zMYXno0JWgmKrjYNnhFMz+bONMu3B\nBKkLfS6pTK4Zn2xBmM7FYypi/IPPM4prKIrwL3barPY6Q3IVi2WEwjNR5Ld7h9RukzEN6kXkTa0Z\n/GrkwbbaMKRzFoIs/DJZhvd29ODCm58PjOkIWxDkC/a5E5uwb2tTYHshwk4bpNaYNBnBCij0uaRE\nF1PMCTx7k4uJ13OILwN82qbKaE6UfUZ8DIKIbiGibUS0TBi7k4he8/6t5SvNEVEbEfUK235ZqnlZ\nLIWg67RKpP4jzVekllakGHGBbHpzLMSCGGyhnHyNdJbhDy8GYzCmWIhKeHLB7jhhK4DftkpPyMVt\nOguCr0wnE7W4T0U6k/WfZdyhyGm+3IoUU2ALcjEJ++h+lUOlIEpZB/FbAD8HcBsfYIxdwD8T0U8A\niE30VzHGFpdwPhZLwfA/cjko6BBpFIT83e1jxMeTinRYXZxDJEq7b06hgjB0Lel4VSWwMc1VDlKT\nqATClhd/tlH0ms6C0AWpectx/rkQUpncmtPxmBSDMEyWV3+Lz4HPM4r7KIoFMVQuplIuOfokEbWp\ntpFrS58P4KRSXd9iKQZp38UUHHdI/UcqC/CtXf24+i9v+LUESguC10FEClLnn3OxXUzpLAsVuplc\nLPI2h3ILBjkULvLKuV/yv11rK6k1a4Ons7k1HaJkHomILqaY3KzQkObLBXwqK1oQ3nmKFIMY8S6m\nPBwLYCtjbIUwNouIXiWiJ4jo2GGal8USgPuRZQsi5qgtCFkGXf/oCjz69jb/uykGoVv5DIgag8i/\nT77zA+osJll3Ra2kBvia1O5nOfuLjwHAZcfNxuQxdcpt/rmE73d85gg/NdiU5soxNdZTkc6yXIZZ\nLOhiMlkQBLfJYDAG4c5T1ZBQRnQx6V4aKr1Q7kIAdwjfNwOYwRg7CMBXANxORMqEYSK6jIiWENGS\njo6OIZiqpZrhrTHkP0iVqwQIC05ZWKuymLiwMcn1KO2+TQVhgFvT8eN/vmM6NHAtTjqCi0luhS02\n3iPB2hKtCQ7Xs421cXz/A/v546Pr4vjyKfMC+/JD509qwpH7jM+b5pqR5lUIYrO+hONETnMFXCEf\ncDFR9CwmcZ9vv3+hcp+KbbVBRHEAHwJwJx9jjPUzxnZ4n18GsArAPNXxjLGbGGPtjLH2lhb1Wq0W\nS7HwXUwRYxCycJWFvqp3T5Sq3CgxiFwWk3qff765BT9/bKXm/Llj5MMzChdTqNo6EAzOBjrdusqU\nfw67iUTlIz7TH513IMaOqlHu60gCV6sghBhElFXnRNxWG7l5qZr16YjHKBik9iRtpBRWb9+vnDoP\nE0fXKfepZBfTKQDeZoxt4ANE1EJEMe/zbABzAawehrlZLAHSGheTrg5CFhvht3G9BRGlSnowdRB+\nd1FVsZ4hxz+dYcYUXPmcqUxwiU2xDkKVHkwaBaESgnyIC9FIQeoo1pcCccnRQgrlADeTSeViipTC\nKsRr8u1TakqZ5noHgOcA7EtEG4joUm/TRxB0LwHAcQCWemmvdwH4HGNsZ6nmZrFEJWmog4gp7PyB\nWBBcjphETpT4Qm4f9XZ+bZWrJdBnSDrBu9v24JZn1hhmJzXyyzLUi2swC49OFdwXBaGoFEztv30L\nogAXU751w2XS2VzbkUTMCcYgIlgQgUI5Q5qrvA6GH9A3aIiharVRyiymCzXjlyjG7gZwd6nmYrEM\nFP7WHXIxOeFgKxB+ww/VFKgqqXmQOkIMIloWk3on/pYtC7fbnluLtdt7/DnI17jxifzGvHjOdFa2\nIEiKQQSPDbiYhM8qIUjSTx6k1q0HkcmIaa6Ft7kVkxQCCwblSQSIO06gtbsqi+n4eS34zLGz8Znb\nlgSO9a0kg5UwVK027HoQFosB/rateutVBQrlN/xQ2wqFhPeD1AYbgsuaKCvK6ZRIKs0X8skCcAV4\nd38a1/ztzdw5tGc3k5FiEHVxMQaRe1bKOgjhq+iCUQlB3x01AAvCtPqbDm51hFpt5NE1NXEncL1c\noVzunm791GEAVEWY+V1MI96CsFgqgZRvQQTHVemaQFg4yyJJJcgyRbIgmLSvTEphQcj7DrTRXybL\n8OKanUhns0hnGOK1uWcTjEGEq6HFGISj+SyP8S1c8URZclSsS7joiJl4Z+sevLjG7MnmzywuxSBU\nsSQOg9uAMNCsz9G7jXSGgsmCsCvKWSxlAH8LDAdW1UHUfAJXGaT26yD0+AFoYyCb/9QpiHAMQt5z\noF06MlmG8298DgAwZ2JjIN+fSBDo+VxMYgxCEdCVHzn36SczWcQcCrnP0lnm32RGcvmYhGxNzEEy\nk80piFjOgmCM5XX1xWOO1GrDsLPmXKbEgErOYrJYRgxcoKtabajeBhmTA77S+QyFcibpnC+FVUQn\nvFQxCFnhDNSC6BNWdMtkWVBBgIxuE12QWmlBeNtzhXfCsZrKdv7sgmmn6jRljl8N7ac5O5GzoQhA\nTYwCLwPGKnnNuCnhyXZztVjKAFM3V90bqChA5BYQqQzD3v40Hli22R/jgsQkdnJ1EPnnnM/FlJYC\nyqrrFMpbm/cI58wiIUg3EtxKqrdirQVhWoKV5c5naqUtKmQxQUBlyYgk/IZ77jEJh3LBbs3zFQPz\ncSnN1RQz0Ckc0zHWgrBYygCdBRFz9H+kotDtk3zjqUwW//rH1/G537+Ctdu73f0VS3rK8G1R2miI\n8ubBN7f4/nk/BpEpvgUh1gtkMizQV4kQdDHJiEOiUlDXQYTHcimk4W26SmpH4yLkcAXHA/tiHYQq\n/HDYrHGBque4Q1oXU8whXHJUm3KOIiYX0xDpBxuDsFhM6CwIXasNIJhy2ZcMWhDpbBZvbenyPgdj\nAibZXMhiQFyJPL1iOy773cv4/An74BtnzEcqza+Xm58snAbbKrwu7iCZYagVo/qBwHP4GF1gWh2k\nVhzvAMiol2zNis36pKwiUwxCtiDiMfKVqS5ALQr0mriDvf1p5b2s+uFZgeO0FoRBCeQrXCwW1oKw\nWAz4efCKwKruDVTMqBF98+75GLr73bF+b1u0IHXhdRA7uvsBABt2ucuhqmIQckxkAKUCAeIxB73J\nNBpqg5XU/Coqt0kgzVXIHVYHqcNj/BiVBSF2cxXvmzQxJPnayUAMwt2me0bi3OIOBf4fmGIGOgUx\nVJlKJqyCsFgMcAEq/wnrWm0AwbYPfVIMIp3Jojfpvln2etYF39+c5sp/RnAxaQSYKgZRrDRXTiJG\n6EllMKom55wQ5ZxK5omCWqwtMbmjxE2mVtri/YiFazHH/IYesiAcwp7+NFZ17NW6hIL1HE5kBaFj\nqALRxjkM9wQslnKGCwhZcOpabQDAO54LCQD6hJXOauJu6mOPpzT4GhFcURgL5SJ0fJX35XAxo6qD\nkN9eB6sgssydY4OoIIReGw5R4DsQdJcELAhDoZw4TR7viDsOHAoGi91mfZ5rSHYxmWIQTlhBAMDJ\nP3lC62ISBXpNzAm4GgcSMygD/WAVhMViIqV5uydDHv2nfptrnSBaEBNG1SCZyfrn6vEUQ7dnUZh8\nTCo3Sb59/e/eT1UdhHy+wa5Gx5XdqNpgJTUnX5qrqHNNzfqCY7kspre/dyae/kZuHTLxfuTmecZW\nFlKaq+ju0llogWB7jHwXou5e8mEtCIulzOECQg7exhyK1JNfVBDjGmuC8YmQBaEnV0kdJYtJvQ+/\ndsYQpC50zQQZbhWNqhUtiBym6mggaEFEzmISWmnXxJ1A87uMZkU50pyfI7uYxH2jWBBxJ2hBhPql\nR2CoUllNWAVhsRjgQjUcgzC7KDi9goKoT8SwfW+//31XdxLb9vSh28t2MXZqzV9L5yPLeNnFJLpa\nim1BcALN+ihYtyCjK3aLakFwpcJjGfGAMM9VPQeD1OYgsJ/m6ruYcqIySpC6Jk7oTw0u4l8GBoRV\nEBaLCV0MIp+AyR2fOy4Rc/Dmplx84tp/LMdhP3gEXX1p7xr68wzEggi3Gs8fgxisBcEJBqnNaa6B\n9SBiZgXB9w24raTFeERB3Z/OYldPEkD4zd+UKmqyIJIZ9frXcjaW6GIaSBdE8T5OWTCx8BMUAVsH\nYbEY0GUY6VptmOCuj/pELGBZcIqVxSRbIlzODEUMgiNaECL5XEw1QmpYXSJ8DpVM52/3qopqsRmf\nnNJrWh6aB76TXu2IaJX0JvPXQYi9m4CBBf/F5/Lzjx6Mnd1JHHXdowWfZzBYC8JiMcBdTLLczFdo\nJdJUG8cLV5/sv5VOHlOnfDs2ZTGJdRCZLPPjFipCHWW976o6iFJZEHU1AwtS18Qd3HflsbjnC8eg\nsTb8/sqFpihv+fE5C0I9J3nBIJOLsMazZLjVIQape5Jp5TGB+zBpn4iI56tLxDCluX7Q5yx4DkN+\nRYtlBJGzIOQ01+g9+cc31qB1dJ1vQdQmYoE1mznGSmp/H4av37UUC655QBuzUL2tMsaUdRBhC2KQ\nlXIeogXhUK6PkcqtIwvqhVNGY7+pY5TnNcUg+Hl0riPZxWRekEd2MeVEZY/C+hOvD4SL/AaSPTxU\naz4Y51CqExPRLUS0jYiWCWPXEtFGInrN+3eWsO2bRLSSiN4hotNLNS+LpRD84LA0nq8bqAjfj79V\n1sad4JKcEeBCP5NluPsVdzn3jj39mn2D34mC2TzGLKYBLKpz3LyW0FhDQp/FZFoPIh+qfbkwVi3n\nKZLJyEreEIPwlHlK6WLSxSDU2VjAwBZiKoc011LGIH4L4OcAbpPGf8YY+7E4QEQL4a5VvQjAFAAP\nE9E8xpjejrZYSoD2rTwbFi5RXUxcWHAFUZdwNBZE+Nq/fnoNvnfPcl+oim/8q7d3Y+LoOu15RJeV\n6F7hFkTbVfeGji0kBvEvB0/DBYdOx7od3Xjy3Y7AtvoBupjy4SiC1DxekE9hpyQLwlwo525LZrLu\ninjCrj0aBSHOSV5neiCUgQFROguCMfYkAPNyTTnOBfC/jLF+xtgaACsBHFaquVksOnSuAFWrjXx1\nEFxI+BYEdzHFY8ogrurSf3h+XWBePOMJANZ43WCBoALLWRu584jtP0xKoBAF0VATw2GzximVlLwm\nNSdfkDofqj35273o1vnr5Ufj8FnjAvvJ1lG0QrmsZ/WIFoQuBiFaEEVwMVW4BaHjCiL6BIAlAL7K\nGNsFYCqA54V9NnhjFsuQovs7VrXayOfSGNuQwNaufl/YJAQXkypDp9BMl7U7cgpCdBX5GU+aKmKT\nG6mQIDWXX62ja0PbxPvL24upEAvCe+SBVhuK7KXF05tDiku+N1McWUxzjUlV1zoLIqAgYrKLaQBZ\nTIoH8/i/nhDoEltqhjpIfQOAfQAsBrAZwE8KPQERXUZES4hoSUdHR/4DLJYC0LmYVK028gm2sQ01\nAMIWRJ0mSP3759/Dd/+xPHhdw/m5L3zpht2Y+637/XGuaNIaBVEsC4ILxIlNYQsi5L7x5qRuwBdd\nQyi7uXoKWHb5mRYc0p2LwxVEOsMiu5iCWUzFsCDCY20TRmkD+KVgSBUEY2wrYyzDGMsCuBk5N9JG\nANOFXad5Y6pz3MQYa2eMtbe0hINjFstg0P0dh7OYKK9g4woi7gep3Z+1ccd3wRw/rwU/Pf9ATGh0\n973lmTWR58rjCn99dZM0V/cnD0YTcsFWwGwl6NpImBjbkAh8NwWhVUZXQS4mQ7tvWSnlVRARFgxK\n+i6m3DZVDYs8N9mCGAjl4GIaUgVBRJOFrx8EwDOc/g7gI0RUS0SzAMwF8OJQzs1iAaLHIKJkMI0d\nlQjsK8YkeJ3AmPoEPnTwNOia9ZiuktZUeecWthEsCDFzyaAEBmJBEBFu/kQ7Pnb4DACKxZUUxwTP\nE/mS/r7iabgwl1NL8wlpU5JBXHQxOUEXkz6LSTg+FIMo3IQoA/1QuhgEEd0B4AQAE4hoA4BvAziB\niBbD/XtbC+CzAMAYe5OI/ghgOYA0gMttBpNlOND5isMxiPznavYtCC+LyVMQRIQGz8WUGyt8rv5C\nQ6E1HYLbV2zbi/f/36f97e4iOur7HEgMAgBOXdiKdV5MRFYCwRjEYIPU+joKuf36oCwIhwepmbem\ndoQYhHC+YmQxlcOCQSVTEIyxCxXDvzbs/wMAPyjVfCyWKGgtiFAMIv8f75h614KQg9QO5dJAfQVR\n4DyJgFQ2V10tItZMAAj0f+L0SryRAAAgAElEQVTjclWxv00a//XF7VizvRvfv/et0L6yfM0Vqklz\nNRyj2t+EslDOXw9CtiDyxSD027iySaWzcBwK7Nubyl9JXZQ6iDLIc7WV1BZLBFStNvJRF3eVgNxE\njgQFUTtAC6KxNu67imTrhimC1CLpLAukvcrbRA6fPR4n7KtuFCc/g5h0nxwSKqkHneaqWjBIkcUk\njuswu5g8CyIbdjHp6yDEGMTgg9TDrx6sgrBYAugtiMJdTHIdRFbI5OFZTDmrojBx0Fgb962AsAXh\n/tTFEzJZhlRarSDkY9ylVTVzk4b5Pci75y+UKySLKTymW5M6XwzC9IbO55TKMLfmRcxi6s+f5ir3\nYhrsSn3DhVUQFouALgah6uaq4rLjZuOofcYDcCumgZwA4+cQFQQXyOLZPv/7l/HNPy8FY8HZiEJq\nVG0cfakMLr/9lcASp4A6zVUknWGBtNfANkW/It2btvwMVO22geC95VsPIh+qdt8DtSBMcPmeyTL3\nfsQgtSaLKbgmdTm8/w8eqyAsFgEuxL9w0pzAeChIrRE+V5+1ANdfeBAuPnImzl3s1nrKFgRR7u3W\nVxCCALp/2Rbc8eL6kCtD9GuPqo1j2cZO3Lt0M155b7c0V3jn1lkJ2eBqZ4pj/fsk/cp58hPgzySk\nAwLrQYSfW2F1EOGxXC8mWUEMXLyJAW+HgjGI7kiV1NFjELd+6jD88IP7D2iepcYqCItFQOcrV7Xa\n0DGhsRbfOXc/NNcn4FBOcIkWBH/BNLke0lkWEMLiG3JtzNEuMMT8ILX+vDoLQkZ2MX35lHnCNikG\nQeo3eYJ47+prREWlYHw3XYFBapNeEg91XUxCDCKKiykePQZx/LwWfNRLES43rIKwWAS4cJWFnKrV\nRj4ch9BUlwhbEML5cxZE+Phv3LUUHcISpbmCOwcxh7TKJZfFpI8z6LKYQvcguZguOapN2Bbclwtk\nU5qrMkhdgIbgh4u3zp9l2ILIoyAMYeCYEEOQs5j29KWUx4hGw2Csl3LCrihnsQjwt/Jwq4jg16he\nkYuPnOm3RuDndluFyy6m8LEPvLkl8J0vx1kTdxCPkdYK4NcxZjFpXEwyjiOtnBcMKAT3FQrnRERB\nLNcUuMdFmkrgGiJ+qw25DsIQpGYsnwURdIuJc+6O1ItJPrkNUlssZc2bmzpx+wvvmXfy/o7lt8+B\nWBAA8JXT9sVpiyYBAC44dDoOmzUOlx4zKxAEBcxvsxw+p0SMkIg56EvpFESwDkImY0hzVREUllB+\nBsQ01+B4PguisPUgwmPxAVoQJsRjZReTjmAvJimLqTjrMA051oKwVA1nX+9WE5v8vTxvSLYgQq02\nBlDlOm5UDf742SMB5ASOyYKQEVt2mFp95Hox6SyIbOQYBBB0AYnCXFZq2joI8VyDrINQrgfhFB6D\nIDLXGYjncpwo6nvwvZguOmImpo6tx0PLtxZ8bKmwCsJiERADySIDKZQzcfKCVswc34DPHr9P5GPE\nlh0Jg/AzWRD1iRiS6WxkF5N73YgWhKEOgivewQapVY9d96tI5IkDGF1MwqE1MSfv2h9AUOGEejFF\ncDF97wP7AYBVEBZLuZLLYpLGJRcTeQLj9EWtWLu9B+9s3VPQdcaNqsETXzuxoGNivovJCfnbRUx1\nEGPqE+hJZvJaEB8+ZBre2NgZuC4Qjieo5ie/yQePKY4FoYzPS4P5Giqa7AKCG5jOMveZRZmjeG9y\nL6ZLjpqV9/hyxMYgLBYB5tcqBDNXdIVyN17Ujm+dvaAI1819PmWBurUFF3iNtXG/mZwKUyX16Po4\nepOZvBbEx4+YiQe+dByAoAA3Nd5zNC4mUQ4PvheTIoahEfSDLVbjz3t0faKgOAkQtCD+ccUxWDhl\n9KDmMlxYBWGxCHCRShR8A5VdBEFXy+AEkXz+0xZOUu7D5zNpdJ1R+Jl6MTXVJdCbyuQNUov3FLAg\nDPedq6QOnisQg1BoiMJ6Mem3yXfLXXJHzxmPBZPDAjrfZfl9j6lPFNwXKR5Iky3w4DJiBE/dYhkY\nWV2FGXJv8oTgG7KchSIKtWI03eTX/cYZ8zGrZZRyn/60m145eUyd2cWU5T/D99lQE/NcTGafuK5/\nks6aELfJAl9+VvLjGkgvJtX8ZCtPVKID+R3xmMqY+kTBSapiFlOhLxCfOdZ1R82Z2FjgVYuPVRCW\nqsO05oH/Ji+5mIxprkVQEP6pKFcZLLOtyy2amzSm3hikvn/ZZmSyTLk6XENNLJKLKagIxBiEuE/w\nGG27bxIU7yB7Maketl88J4nxREBBuJ9HeV10T1nQmtdtxM82pj5R8II/YgwiyuJSImfsNxlrrzvb\nX09kOLEKwlJ1GDtrChaEKCRDaa5OWPgMBrEVhU748/5JU5rrjEKnqy+N2198TxmDaKiJoyeVzhuk\n1hkoAcURNc1VskDkWRXq39ch/1qD/ZTcnwdOb8ba686OtK4z74U1pj5RcDfW4P+Pgg4tK6yCsFQd\npmU1AzEIUUEYMmSKuXawQxQqsuIcPmscAKC9bZzWyuDs7UsrLaW6hGtB5FUQmnsKBqmD2/R1EGZh\nORABKv46dAomprCCghlZ0RhdHx9UoVs5rC09UEqmIIjoFiLaRkTLhLH/IqK3iWgpEf2FiJq98TYi\n6iWi17x/vyzVvCyWjOFtMBeDCC5Ub3qBLOYbIhFphf8HDpqKNf9xFqY21+etEm5uSGgsCFdB9Gla\nVnO0S0BoXE+AUAchp7kGvHGDC1KbML3j80vENQF3EwOJQYgU6mIqJ0ppQfwWwBnS2EMA9mOMHQDg\nXQDfFLatYowt9v59roTzslQ5xiA1cpXNoqAzuRiK5SIB3LdaXYaSQ7lr5VMQWca0CqInlUFnb8pf\nzU45jwJbS7jHqMcBMQahOk/hz091iGm5WMe3IHL3nO+qM8c3AACmjW3Q/v7FeYwb5cYMPnDQ1MA+\nI9mCKOWa1E8SUZs09qDw9XkA55Xq+haLDmOQOmIMQqQ4WUy5SmOdi6mQVg69yYzyPutrYmAM6NjT\njzH1CWzb0684OlorkXC1OVOO51M2g5WfUY7nvyNTcF/m3iuPxd6+NCaNqcNr3pobNTEnkCIcd8jP\nCBtTn8Bb3z3DXyjKv7a1IAbEpwDcL3yfRUSvEtETRHSs7iAiuoyIlhDRko6OjtLP0lJxmC0IF6Kg\noDMcUqQ6CO9cjt7FJF4ln9uiL5VRWxDeSnabO/swpj6B2RPUKbVR3CLybfvdamXLQjru+Hktge/F\nEqCmdhbKGESe31tjbRyTxtQByCm/kPCXzlFfE/PPy1cNHMH6YXgUBBF9C0AawB+8oc0AZjDGDgLw\nFQC3E5Gy9JAxdhNjrJ0x1t7S0qLaxWIxYo5B5LqrBiup9ccUM4uJiJDQuH7E64hvwvd84Rj/80vf\nOgWAuyymKhBd76V5bulyFcR9XzwWnz1+dmi/Bm8/E7KA9S0IKd1VfjyfOrrNn6d7X3kvFSIQpOYq\nKEKcKGoMYrZUi8KVX0NN0OkyfZzrhjpmzoTQOUbXxwPHjkSGXEEQ0SUA3gfgY8z7q2OM9TPGdnif\nXwawCsA87UkslkFgymLa3eMtBiNZEKYgdTFczGIzO1H4v3j1yb5vW7cgjZiyObYhgabaOHqTWf9e\nRAFc7wm4LZ4FUZeIobk+nG/fWJff+yzfdk65yvuFXU4tTbX+98EqWMqvH/w5RIlBPPClY3HAtObA\nGP/91EuKs238KDx71Um44sTgErUAcOZ+kwHoXYYjgSGdORGdAeDrAM5hjPUI4y1EFPM+zwYwF8Dq\noZybpXrQKYhkOov3/V+3Jbgbg8htM7svijc3AgW6kE4cXYfm+gSAfAvSuMQcQl1NDL2pNHb1JN1z\nNNXhtWtOxSv/fioaa10B15/OYox3XpX8qo2HLYgJjbWB77JgHz/K3X6ol47r3xNFyzAqBPGYuV7F\n8fxJTYF9Jo1x53PIzLG+clUF9y88bAae+NoJ/vepzfWhffh/Ge424sQdwpTmeqWb7N/OXoAnv3Zi\nQBmONEoWpCaiOwCcAGACEW0A8G24WUu1AB7yzNPnvYyl4wB8l4hSALIAPscY21mquVmqE17Rq1MQ\nvULqJ1FwFTExD152vxS3DiLsk1e9zeuWtCQi1Cdi2NOXxp6+NC47bjYuOmKmX5U7sanO33e0QvHo\nePaqkzCqNjgPWSa2TRiFh79yHNrGu+4ZgqsYxN2KlcUkcvKCVvzzS8dhXmuwNcWciU146MvHYXZL\nIy75zYsAJMVKOVfYzPE5l5JyPkxtQZhiNfGYgxleJtRIpZRZTBcqhn+t2fduAHeXai4WC+D+4WcY\n81NAxbRRAEgLPnuH9M36SqEgdOtQAG6wVN5mSnOtT8SwpbMPgPs2zP3kADBtbO7tmCuIrV25fTfu\n7lWec4rirVp123MmNkXaT6QYz2/fSeHrAsDc1iZvDuH0YP5J16lXJBeDiK4gKoGR6xyzWAqE/y1v\n7uzDPlffhz9Iy4+KaaFuFlNum2h0yG+RRUlzFa4rw9/cxW2mbq51NTFf0Dc3JALbuFsJgO+64n75\n8w6ZVtCco9d/mPcbChnLryHGIKY0u9bUrAnBt3zVbfEAvOxisgrCYqkQuEBbtW0vAOBvr20MbBcb\n2LlZTGKQOqch5ABmUQvlDBaEuAa1qQ4i4RA2exbEWKnhm3j+Q9vcWMGVJ8/Bnz53JI6dG87EMc41\n33bBhRNlv1LCryAq1pPmt+L2Tx+OTx8TzOJSKwj3p2hByFZmJWIVhKVq4H/L/I9ddi30iwqCgoKC\n79s+cyx+fN6ByvMOBrFZn8woL7Dc3Z/2x2oMFsSGXTk3kawgRBZ5i9g01MRxaNs4NNUltPuqyOca\nErOazGnCBV12QDgKFxMAHDVnQijmo7ovHmieNaExsN9A1iYfSdglRy1VA//D5+4CWWTJLbDFt0N+\nzFFzJoRcTMV5A1ZXIQNAY60ruPcKCsLUjXSLF1MAwi4mAHjkq8ejuz8dEoxNEVJbRaLedrBdePig\nQmIQ3JqaPraw4G/UFiWA2jJ6/wGTURMjnLygFT97+F3vnEBskKvWlTuR/0cQ0TEA5jLGfkNELQAa\nGWNrSjc1i6W4yOsZy2+1YgsFd8lRMUhtOu/g5yb3Krr780f6aaNccIsKYppBQNYnYuhNZXD9hQcF\nAtScfVrUC9EUqiCiCnaCGwB/e8ueUCVyIecB3GD0Lz9+SMHusHGjgvEWE8plTYlwhlfXII5ZCwIA\nEX0bQDuAfQH8BkACwO8BHF26qVksxYX/LesqqYMxiCDcLaUSB8VMc+VvuofMzNUStI52g6l1Um3C\nP644BslMuCvrPVceg827+3BMgUJ0lFdEFzVvP1JsgTEQAT+9YDGefLcDsxXKqdDHd8Z+6iVZTfBn\nqFpEaaDzqYYYRNRXhg8COAjAKwDAGNtEROq8MoulTOF/zLwOwuRiIikFlvdvUgmPYr5EquTNhw6a\nir5UBh9uD2YZ7T9N7Wbap6VRayUYr+0QbvjYwZEW0wGiu9aI3Oyp9x84RX3dIXgLn+gpiA5Nc0KR\nqPcVd5yCmv+NRKIqiCRjjBERAwAiUnf4sljKGP6nrGvWJ76Ny77yrNCjSaaozfoU53IcwsePmGk8\n/h9XHIOdXuX0YDhz/8n5d/KI+vKsemYDOc9gaPWsoq1CfGaw/Oe/HICFU5Qt4yqGqArij0R0I4Bm\nIvoM3E6sN5duWhZLcbnmb8uwy+tNpHcxBesgRLmVMVgQxSmUy/ViGgg6a6KU5BP8FPqgZigsCF7o\nZ2r1XihnHxBdmY5UIikIxtiPiehUAF1w4xDXMMYeKunMLJYicttz6/zP3IKQ9UQgSA11mqs6BlGk\nSWqvUJ5EtyDybB+CW140ZTSuPmu+1s1lUZNXQXhN9B5mjJ0Id0U4i2VE49dBSONyDEKEWx3qGEQx\nXUyDPtWQET2Ym8fSGIpCOSJcdtw+RTnXLz9+iDIbqxLJqyAYYxkiyhLRGMZY51BMymIpJdlcnmtg\nPFgHoYlBKFMgBz8nUy+mciXqSnEj6JYiMZAsqpFK1BjEXgBvENFDALr5IGPsypLMymIpIfoYhNjN\nVe1iUlHUGMQIejGNXgcxMjTEXy8/Gs+u2j7c0ygroiqIP3v/LJYRD9O5mDL6OgjTIkPFdAuNFGEK\nRI+W6PTIn//PUXj8nfJZNnjx9GYsnt6cf8cqImqQ+lYiqkFulbd3GGOp0k3LYjGztasPNz25Glef\ntaDgYiXTgkEcIgoI62yJYxC5cxXtVCUnn7VD3ooQuls6eMZYHDxjbLGnZSkikQxaIjoBwAoAvwDw\n/wC8S0THlXBeFouRr921FL9+eg1eWL2j4GP5ug+hLCZTJXXWVAdR8BRCmOogypXI1s7IuSWLRFQX\n008AnMYYewcAiGgegDsAHFKqiVksJnwhP5BjNRZEf0bfzdVvtVGiOgiMwCB11N5NI8ltZgkSNSSW\n4MoBABhj78Ltx2SEiG4hom1EtEwYG0dEDxHRCu/nWG+ciOh6IlpJREuJ6OBCb8ZiiUI6w1ttBBVF\nSiqUE8kIratliltJPehTDRmzJkRrqDCCdJ5FIqqCWEJEvyKiE7x/NwNYEuG43wI4Qxq7CsAjjLG5\nAB7xvgPAmQDmev8uA3BDxLlZLAWRympcTJngmtQizBiDKOLkRpAwHTdKv9YEAP9eRtAtWSSiKojP\nA1gO4Erv33JvzAhj7EkAO6XhcwHc6n2+FcAHhPHbmMvzcNt6VH4tu2XI4RaEjByDULbaUIi74tRB\n6NeDKFeiN+sbOfdkCRI1BhEH8D+MsZ8CfnV1tJ7AYVoZY5u9z1sAtHqfpwJYL+y3wRvbLIyBiC6D\na2FgxowZA5yCpZrJaFptiIpDFmqmFj6lbtZXbpy2sBUTR0f/8x8Bt2TREFVBPALgFLgFcwBQD+BB\nAEcN5uJih9gCjrkJwE0A0N7eXrzOW5YRhalwLR+pjHpNgKxwUgICks2U5lpMoT4SYhA3faI90n4k\n/bSMPKK6mOoYY1w5wPtc2Jp/ObZy15H3c5s3vhHAdGG/ad6YxVJU0pr1IEQrQZb55krq4sxLdd1K\noBLvqVqIqiC6xawiImoH0GvY38TfAVzsfb4YwN+E8U942UxHAOgUXFEWS9GIZkHkog0xh4R236oY\nRDFabRTvXOVHJd5TdRDVxfQlAH8iok3e98kALsh3EBHdAeAEABOIaAOAbwO4Du76EpcCWAfgfG/3\n+wCcBWAlgB4An4w4N0sVMhg56qe5SmYB01gQMYeMaa6czx0/8G6hPOV2JMQgolKpzfqqCaOCIKJD\nAaxnjL1ERPMBfBbAhwA8AGBNvpMzxi7UbDpZsS8DcHneGVssg0TVamNVx15s7swZxeJ6EDEify1j\nnbBbe93Zg5pTrpvroE5TllTgLVUN+VxMNwLg6xgeCeBquO02dsELFFssI42UYuH6k3/yBF55b3du\nQJBqcdHFVOK5VWLVcWW6zaqDfC6mGGOM1zFcAOAmxtjdAO4motdKOzWLpTTkXEz6fURB7TgktNoo\njbDjU6kkWcqfYQXdUtWRz4KIERFXIicDeFTYFjV+YbGUFbogtYi4JrXYLbZkAnwE9mKKSgXeUtWQ\nT8jfAeAJItoON2vpKQAgojkA7OpylmGjkDoIORjtF8oZWv25MQhXsg2l0B5JCwblgz/fSnSbVQtG\nBcEY+wERPQI3a+lBlvtLcwB8odSTs1jyEUX0yMqkJ5lR7yieV1AKcdGCiDqxAVKJwtRaECOXKGtS\nP68Ye7c007FYio+8xOjG3W62kjEGIaW5KjcUkVyaa0lOPyxUorKrNirIoLVUI1E8TboV5EzHis36\nRLdPyUIQFVwoV4G3VDVYBWEZ0USJRWQH0LgpYEFQ6YPUI3E9iKhYS2LkYhWEZUTCBXUU4a/rxCoH\nr6Ur5ArlAjGI0gq7SrIg+K1UUuC92rC/OsuIJoqC0LmYTOhiECWzIFjlxSA41oIYuVgFYRnRXPKb\nl/C1P72Otqvuxcpte5T7ZDUKYlVHNy6//RXlNlGkiWmuJS6DsHUQlrLCKgjLiEQ0HP708gYAwNMr\ntiv3DXRplYTVvUvVDYOJcv1cE7Gh+zOpJGFq14MY+VgFYakYdvemlONimms8og/HEdKY4rGhcDHx\n61aeOK3AW6oarIKwVAy7e9QKQuzNF1UAi35z0YIofZC6pKcfJirypqoCqyAsFUOnxoIQXUyxiBaE\nKKgTggVRallnLQhLOWEVhKVi2N2TVI6LWUxRFYRIzBEtiNJSScKUp+xW0C1VHUPekZWI9gVwpzA0\nG8A1AJoBfAZAhzd+NWPsviGenmUEo4tBZAcQgwh0cw102iituKtMC6Ly7qlaGHIFwRh7B8BiACCi\nGICNAP4Cd4nRnzHGfjzUc7KMPFQyRxeDCFoQ0YxmChTKDaEFUeLzDweVeE/VwnC7mE4GsIoxtm6Y\n52GpAJJp9ToPYhmEKmNVVVEdLJRTj5eCSrIg/DTXyrmlqmO4FcRH4K45wbmCiJYS0S1ENHa4JmUp\nf1QF1LqK6UCQWiGtVOfSdnMtMZWkIDi2knrkMmwKgohqAJwD4E/e0A0A9oHrftoM4Cea4y4joiVE\ntKSjo0O1i6VK0bXdCLiYYmFhJbcDBzwXE8ILBpVaftNwv7KVgArUeVXDcP53PBPAK4yxrQDAGNvK\nGMswxrIAbgZwmOogxthNjLF2xlh7S0vLEE7XUu7oFEQwSB3+L686jkhoNhdotVHiOoiSnn2Iqaib\nqU6GU0FcCMG9RESThW0fBLBsyGdkGdHoevIFC+XC25UuJuFzfCjWpPaoSBdT5d1S1TDkWUwAQESj\nAJwK4LPC8I+IaDHcvmVrpW0WS150MYjMAC0IjmNjEIPCprmOXIZFQTDGugGMl8YuGo65WCqHdEad\nxSQqDpWwV+sVIc01EIOwrTaiYpv1jXwqMCRmqVZSGc3SonkK5bQxCB6kHoIlR8XrVhqVeE/VglUQ\nloohmckqaxrytdpgCsND3GsoFgziVKSLydoQIxarICwVRVrhL8rkadantiBy+6lqJ0pFRSqIyrul\nqsEqCEtFkVLEIVigkjpqHUQOZwjXpK6kJUdts76Rj1UQlooilc7jYlK8zuarg4gNZaFcBb1u++6+\nyrmlqsMqCEtFkVRYEPlcTOo6iNx+QQvCUig2BjFysQrCUlGoXUz50lzV2U8XH9kGANh/6hh/rIJe\n8EsOt4YqyW1WbVgFYSlLXl+/G1s6+wo+TqUgMnkqqZ9asT00xsBwysJWrL3ubLSOrhO2lEbaXX7i\nPiU5bzlQSW6zasMqCEtZcu4vnsHx//VYwcepFYRgQSiE1dfvWhoa0wW2SyXrvnb6fKy97uzSnHyY\nseph5DIsldQWSxT6Nes7AO4bvoqkEKRes70b9YlYwIUU1d2hW8faCrvoWMNh5GMVhKWiEIPUJ/74\ncQDAzz96kD8W1d0hqp+oy5RaLJWGdTFZhpXVHXvRm8wUfJwuMya/iyna+UUXkzOEvZgqEU2TXcsI\nwCoIy7CRzTKc9JMn8Nnfv1y0c6YUbqmgiymiBSH2b4pZF9NAsM9q5GMVhGXY4PUJT75bvJUBVXUQ\n4noQUQ0A8a13KFeUs1jKCasgLMOGbv0GVcO90D6S4yLhveWrOrqKhXKRYxBDnMVksZQjVkFYhg2d\ngtCtDCci65C6RAxA/hhE1MZ7zQ0J/3N8CHsxVSJRFL6lPKl6BdGfzqA/XXiQ1DJ4VE3yACCd1ae3\ncuRD6w0KQgyC5wtSX3HiHDzxtRMCxXGB6murHyJz9JwJAIBa73djGXkMW5orEa0FsAdABkCaMdZO\nROMA3AmgDe6yo+czxnaVch6LrvknGmpiWHrt6aW8jEVBVrdEaAQTQm6PUV/jCqGkIkjdk0z7n/MF\nqWMOYeb4UYGxuK2DGBA//vCB+PKp89BYa7PpRyrDbUGcyBhbzBhr975fBeARxthcAI9430tKOsvQ\n1ZfOv6Ol6GjXkB6IgvAtiPCx3YIFMZA01Upco2EoqEvEsE9L43BPwzIIhltByJwL4Fbv860APjCM\nc7GUmMEpiOD32iK5mFQELAirLCxVxHAqCAbgQSJ6mYgu88ZaGWObvc9bALQOz9QsQ4E+BpEb/8Vj\nK9F21b2hOJF8ZH3C/a+sUhDd/TkLMZ98V83Itvu2VCvD6Rw8hjG2kYgmAniIiN4WNzLGGBGF/l49\nZXIZAMyYMWNoZmopCdosJmH8pidXA3CtgNp4LtgpZ8ZwF9P3730LyzZ24qfnL/a39QygUlskbtNc\nLVXKsFkQjLGN3s9tAP4C4DAAW4loMgB4P7cpjruJMdbOGGtvaWkZyimPKH711Gq0XXVvJHfNcKFL\nVhItCC6Q5diCHIOoEzJl/vrapoB1IgapB0LMprlaqpRhURBENIqImvhnAKcBWAbg7wAu9na7GMDf\nhmN+lcB//fMdAOqsnnJBl84qKjUujmUXk3xoIuYEBPncb93vfxaD1KJeERcCMmErqS3VynBZEK0A\nniai1wG8COBextgDAK4DcCoRrQBwive9LNndk0TbVfcWtU1EMeFyULdaWjmgm5tq/Qa59bd8bMwh\nv5paRmdBnDh/YnhQMSeb5mqpVoYlBsEYWw3gQMX4DgAnD/2MCufNTV0AgBseX4Xj5pWhq8uTc+ky\ndjEp4skA1C6m/lRwZ1mOuwrCQV9KVQehjkFEzWiyhXKWaqXc0lxHDCNFTqR1UrgMiJLmSr4FIbmY\nJA0Rd0i7bkNPf0bpGoradsNiqVasghgkupXNyoWyDlJHaLWRi0GYXUyOQ9rfRHcyjVE1rrEc6NI6\ngKIIG6S2VBNWQQwUT06UsYsfQHm7mHRzEwPQXBHICiLkYiLS/i56kxm/FYfIQCqkrdFhqSasghgg\n/E2yXMUvt2zK2YLQzU20ILgS6U/lcTHFSNs1NJ1laPAURLCNd8FTtvaDpaqwCmKAlPubJCujIPVv\nn1mDXz+9JjQeJYspk3PDJM8AABn+SURBVNFYENIxiZhjVNZ18WJZEGX+i7dYikhVt1kcTJ/6ck4f\nFRnuIPXqjr249h/LAQCXHjMrsC1KkNq3IPLEIBIxMppztYnwu5BKQeT7rVr9YKkmqtqCGMzLtS/E\nylxPDLcF8cbGTu22KAoi4ysIc6Fc3DH/V1ZZELEBBaktluqhqhWEqrFbVNKe68NmMZnpNfRBEuf2\nnw+8jZueXAUgqNR4PCJcBxG2IEx3yi0I8fc1kM6u1oKwVBNV7WIajPDkQqxcPU18WqWyIJ54twN7\n+lJ43wFTjPv1pgwKQnh4NzzuKoePHT4zMM6nH3YxBc8VjzlGl2GdYlWzgaS5WizVRFUrCFF4MsYK\nCkAOt28/H1xYZiIs3xmVx9/ZBsbcFhUX3/IiAORVED2BPkjBZ6xaUW755i4/MC0SpVDOGKRWKQhV\nDCKvPrVKxVI9VLWLSbQg7n5lI97ZsiewfdPuXtz67FrlsSluQZRsdsVBtcLaQLnh8VX4n0dWFHRM\nn2BByG0wVBbcso2dSqsnnwVRE3eMwr02Hv6vXkglNU+TtS4mSzVR1QpCzLf/1z+9jtP/+8nA9ot+\n/QK+/fc3sas7GTqWv5n3JDP41VOrh93XzxjDb55Zg66+VGBcNa9nV27HS2t3Yk9fCr96arV2bWiZ\nZCaLrt5U/h0BPLtqO15cszNgQchN81Rz27CrVzmeLwYRdxxjPKiOZzEJuxTiYuLrTVj9YKkmqtvF\nlOfteu2OHgDht1cg92b+1uYufP/eLkxorMUHDppa/ElG5PnVO/GdfyzH6+t3478/cpA/rnob/+iv\nXgAAnHPgFPz99U3Yb+oYHDF7fN5rJNNZdEZUEB+92b3GRw/PLerUk8xAvIpqRblkOqscz+tiiukr\nqQFdHYR+f5n6mhjQbesgLNVFVVsQ+d76+XZVoFU+ts8QjB0Mmzt78eCbW/Lux6+/pasPd770nu+C\nMcUglm1yU1BV/nkVqYyrIFTB4F3dSfz11Y34wwvrAkt89gUsiNznTJbh9hfeC52nP51Rzjl/oZxZ\ncPN7FI9TpbnqrBDfxWS8isVSWVStBfHejh48s2p7pH1VqZpykLpUL5bn3fAcNu7uxZr/OCvS2+vz\nq3fi+dU7/e+ilbRuRzc27u71v2/Y5X6O6h5LZRjSWaZsn33l/76Kp1a4z/PPr2z0x8V9H3l7K2rj\nDtomjMJdL6/Hs6t2hM7Tn84qLbt+z3p5esV29KczIZdTvkpqVQyiEGuAu5h062hbLJVI1SqI0//7\nSWMKpohqPzn4W6oun1ygJzPZwJrMURFdTGf9z1OB1dX4anNR60H4/nKcAwA27sopnpfX7fI/i8/u\nRw+8g7c278H/vfAg7O7JneOofcb7yqI/lVVWqfenMvjGXUvxgMaaijtOwZXUqiC17vfY4HWDNdV1\nWCyVRtW6mPIpBzFwq3IfyW/dpXZNq+IgIjrXiKggujXCzaQgGGN45K2tSGeySHr7iXEI/zlp7r83\nmUFTXe49ZHXHXm++OeoFF9fSDbtx23PrAATdRv3pLNbt7NHO0y2UK6wOopBmfdzFpFt8yGKpRIZc\nQRDRdCJ6jIiWE9GbRPRFb/xaItpIRK95/84a6rmJ7OnL+dFVb42prOxiKq2GkF0qMjrPh+jPb1C0\nvAbMCuLxdztw6a1L8P8eX+Xv1ym8/XOlobv73lQGExpr/e9rtneHYhii8N7U2eev1tfcUOOP96cz\nqFG4iThuoZx2sx+kFq+t+p3plEydryDUy5daLJXIcFgQaQBfZYwtBHAEgMuJaKG37WeMscXev/tK\nOYmxDQnj9p6UoCBUFoTkYhpM478oyFk8Mrp6B9GfP6W5XrlPMq2f+9bOPgDA+p09votpl0pBaBRk\nTzKN8aNqhO8ZbNvTH1AoKvcPABw4rdn/3J/Ootbwyp9wKNQMMLA9Hm7Prop1HDtXvXzsuQe6BYGL\npozWXsNiqTSGPAbBGNsMYLP3eQ8RvQVgyPNDJzTWBgSdjOhKUMYgJBdTssSV1flcTLrri66wCY01\nWLktvI/JguAuqniM/P069vT525dt6MRRcyZoU0b7UlmMb6wJjK3u6A5812VR/eTDByKVzeKqu5di\n0+4+9EM/z0TcwVVnzsc3zpgPBuCwHzyMHYr6FZGpY4MK87VrTg1YLSKnLZqE1T88y7bnsFQVwxqD\nIKI2AAcBeMEbuoKIlhLRLUQ0VnPMZUS0hIiWdHR0DPjao+vNFoToVoqSxZTKI8AHSz4Xk+76oiLT\n1X2YFARXMA6Rb6Vs29Pvb//or15AXyqjDe72JNMYL7iYANfNJFKvURCJOGFCYy1q4zH0pzPYvte9\n7ui68HtN3CEQERyHEPM+52Px9GYs/+7pGJPn/wLHKgdLtTFsCoKIGgHcDeBLjLEuADcA2AfAYrgW\nxk9UxzHGbmKMtTPG2lta1O6AKOTL3MlnQchB6tJbEGYXk9aCEMZ1VkgUBSHe7tauvsA+vcmMNkjf\n1ZdGY21QoK/ZvjfwvU7jYuJ1CrVxB32pLHZ0J3HpMbPw0r+dgjeuPS2wb0JyP4myfPyoGmN2Et+3\nDNZWsljKimFREESUgKsc/sAY+zMAMMa2MsYyjLEsgJsBHFbKOeR7IxeDkcogtfQ2noxoQXT1pZSt\nO0T6Uhls2xMUwnldTJrtYhaTTskkDRXlXEGIMZYOwYIAgG5D4DaTZSELYc327sC8VFXOQG6Nh9qE\ngy1dfchkGaaPrUdtPIamugTe+u4ZvnCPS4VyvBHf7Z8+HA995Xh/XBUqOnP/ye48NIrKYqlWhiOL\niQD8GsBbjLGfCuOThd0+CGBZKeehE5b8bVpUCuo016BAjqogrrp7Ka644xXjPpf85kUc9oNHpPkO\nTEFkAgpCY0Foxrfv7ccer+ZBPM82SUG4FoTe/VIvZE8lYoS1O3oCc9HFILjwr43H/OtPaMq5q+pr\nYr7lIFsQJy+YCABYNGUMxo2qMaYhf/ecRVjyb6f4tQ4Wi8VlOP4ijgZwEYA3iOg1b+xqABcS0WK4\niSZrAXy2lJPQCcveVAaJmKN1MXXs6UdLU20oSN0vuWm6+9OIORQSfqu2dYdSZGV4JXRAuBvqNhhj\nIbcPJ2BBaKwmlYspncmi/fsP+9/F57G1S7YgMsa+RmJ67dTmeuzY2x9Q0HWa9FuudMQq6AlSPIMX\n1SWkFeWuPWcRLj9xDsYostVe//ZpAYsoHnNC57VYLMOTxfQ01GnzJU1rldEpiL5UBqPrEr5SiDvk\nWxNrt3fjhB8/ji+fMi+U5iq/wS/69j8xtbkez1x1UmB8c2dv5N5He/pSIHLdIiYL4rbn1uFXT69R\nbhMD0/msJk5fKhPopwQEq6d5sJjT0582vqGL9zt1bD3W7ugJWGhyG4xr378QJy9oVW5vaQoK8oyQ\nZSWSiDnatN6oQWmLpdqpWpta90bel3SF5e4eN04wvrHG97Gv3OYGV3/28Lv40MHBzNxkOoveZCbg\nThH7HgFuXKOrL410lqEvlUFdIgbGGPpS2cBxHLFi2eTCevitrdptoitMp2TkGMT8f38gULsAuMFm\nHT1JfRYTELYgAGBLZ87ikRvtja5PYPq4Bv97raBgwhaE+1NWEDpsHNpiiU7VRuV0wrI7mcYzK7fj\nxw++CwBoGz8KWzr7kM5ksakzJ/DlIPUfXngPC655AOt39miL5rhQ7ElmMP/fH0BPMo3/fngFFlzz\ngLJCt7M35YtdkwWhWhmNk84Tg4g7pFQ+cg0Bj0Wo3r67k2ll/ySOGKSe2uwK/s2CgpCVi9xlVbQg\nVCmuQNjFZLFYBk9V/lWlM1ntWs1bu/rw/Opcl9Hp4xqwbkcP5nzrflzztzf9cW5hyPzyiVWBt23R\nVSO+NQPAsyt3+Cu0bdgVtDYAT0F4wt+U5mry/3MXTDqTRSbLcOVJc/DJo9v87bVxJ1Kzvq5e9z4m\nNoV99T3JjNHCEa0jXpy2WVC2cnsLnYKojTvaYHjC0IZDpNQV7xZLJVGVCoLXDHz11Hn4xhnzA9su\n+c1LeGh5zmUztbleWZG7U5Oq+vqG3TjwOw/63zd39mLDrh60XXUvfvf8usC+dy5Z73+W3VFAdAvC\nlEF045OrsWxjp398Y10cRwqLAyUkBaFTFtyCUC0s1N2fNtaBiBbENE9BbN+rT/WNhxSEe7zs9jId\nY7FYBk9VKgiezdNUF8excyeEtr8trE09VRPo3LE3iX1aRoXGl23sCnx/Y2MnXnlvNwDg/mXBVtWi\nItokKAj+Bn3F7a/6lo6YgZRMZ3Hcjx7Dn1/ZACD/IjbPr97hK4jaeMxvehdzCImYg9ueW4e2q+5F\nMp0NNCkU4cefOD9cnPj9e9/Cuh36Tqti+qjueYqMldpd8F5NckW2iJzmarFYBk9V/lX5wjIR84Wx\nLrNlmtSvZ/6kJgDuym2bdvfhzsuOCGXWiHz5ztfxy8dXabcf2jYWcYewcVcv7nzpPbR//2H1msye\ni+n0nz2JD9/4HN7b2YM7X1qPw3/4MF5+b1dof5GYQ/7xtXHHVxDN9QnUCIKVr1Mt8vUz9sUpXk0B\n4D6n2z9zOP73siOM1xSpT8QwZ2IjAGDymDqtJXD6olb88uMH43DJSuHCX+7pJKJaHU7ELhVqsRRO\nlSqInLDkLpVjJEuiJubgrs8diUNnjcO/v2+hP370nNx+vakMDp893hd4h7WNC5zjTk+ILt8ctCpE\npo9twOTmOqzb0YNv3P1GKIWUs7s3hf2v/Sfe2boHr693LZIX1uzE1q7+wOI7Krbv7fctkNqE4/v0\nxzQkAi20H317mx9r4Jx3yLRAFlEi5uCofSbgiNnjccPHDtZeU8xMqq+J4fbPHI5bLmlHPOZg0dQx\nymNq4zGcsd/k0HiXl801flRYETfn6corYyMQFkt0qlRB5Nwt+08dg+s+tD/+67wD8Pcrjsbi6W6L\n6UVTR6O9bRwSMQeXHjML53jtng+ZmesheNfnjgSQc0n9nxP38bf99PwDcfjs8VgwOdgeWg7yto6p\nw9yJTbj3jc2B8XmtjRgnvGnf/sJ7WvdPPrZ19QfumYcLxjbUBAT5c6t24P0/fzpwrNxHSVQovEUF\npy7h4L4rj8UvP34Invjaif54fU0ME5vqcNJ8t7Zh/6nBZzLKc0GN01gWXKjPVrj07vnCMfjVJ9qV\nx1kslsFRnQqCv017WTEfOWwGGmriOGBaM649ZxEAhITxtecswg8+uF8gZtEuWQzHzJmAv15+NG7+\nRDs+dPA0AGGFILus6uIx7Kd4o77g0Bn45pnzQ+Mc0+I5nN9ccigA4E8vb8B5v3wWgHvPvBPtzHEN\nfr8jQG3p1Cdi2NYl1iwEr3vzJ9rxvQ/sBwCYNaERC6eMxhn7TQoUqcm9mM5dHKwhOXVhK7537qJQ\nwgDnvEOm4dvvX4jPHDs7tG3a2AacsrBVcVT4PgBgdJ0tkrNYolKVhXLNDQlcfORMzBjfENq2eHoz\nfnbBgZg7sSkwPm5UDT52+ExlmuTdnz8Su3tSiMcc3wLhcAVx9v6T0dJUi/Pbp+OltTuxbkcPbnlm\nDcbUx9E6ug4AcOTs8XjOS7EdVRPzhez7DpiMe5YGLYyz95+Mv7y6EU21cezpV1sWJ86fiHGjarCz\nO4n3HzgFTXVxHDZrHEbVxPFvZy/ABYdOx1nXPwUAWDh5tFJBEBG+94H98NjbHSACZo0PvsWfurAV\nW7v68O9/XabNJJLjA/Nam/CLjx6M3b1JTGisBRHhoiPblMcCrlL65NH6xYCicPL8ifi3sxfgwsNm\nDOo8Fks1UZUKYvq4Bnzn3P202z940DTtNiLCj/7lACyekVMEh8wcp92fC//TFrX6b84Lp4xGbzKD\n5oYELjx8BhgDrjx5Lj51dBuO+c/HsLc/jfqaGI6cPR5XnTkfHz5kmq8gjp07AU+t2I4T9m3Bs6u2\nY+b4UTjnwCk4fNY4EAGvvLcbM8Y1YK9nAd140SFYv7PHt2g4n/bextfvdLOnvnb6vnji3Q789tm1\noXuYP2k05k/Sr6TWOroO3zlnEU6aPzEw/uCXj8NrXrxE5uwDwrGGUuI45N+zxWKJRlUqiMFy/qHT\nI+87cbRrQcye0BgYr6+J4cqT5/rfv3LqPADADz+0P55btR1Hz5kAxyF87ng3rvG7Sw/D7p4UTlnQ\nip8/tgKnLGhF3HHQVBfHcfNyqadzJMvn0LZxOLRNr8A4x81rwYnzJ+KTR7fh/mVbcMDUMcriPR0X\nH9UWGpvX2oR5rU3hnS0Wy4iARnJlaXt7O1uyZMlwT8PI1q4+/PbZtfjqqfMQL8Nc/YeXb0V3Mh2K\nC1gslsqFiF5mjOXN7rAWRIlpHV2nDb6WA1ECvBaLpTopv1dai8VisZQFVkFYLBaLRUnZKQgiOoOI\n3iGilUR01XDPx2KxWKqVslIQRBQD8AsAZwJYCHcZ0oXmoywWi8VSCspKQQA4DMBKxthqxlgSwP8C\nOHeY52SxWCxVSbkpiKkA1gvfN3hjFovFYhliyk1B5IWILiOiJUS0pKOjY7inY7FYLBVLuSmIjQDE\nMuVp3pgPY+wmxlg7Y6y9pSW8eI3FYrFYikNZVVITURzAuwBOhqsYXgLwUcbYm5r9OwCsU22LyAQA\n2wdx/EjE3nN1YO+5OhjoPc9kjOV9wy6rSmrGWJqIrgDwTwAxALfolIO3/6BMCCJaEqXcvJKw91wd\n2HuuDkp9z2WlIACAMXYfgPuGex4Wi8VS7ZRbDMJisVgsZUK1K4ibhnsCw4C95+rA3nN1UNJ7Lqsg\ntcVisVjKh2q3ICwWi8WioSoVRKU2BCSiW4hoGxEtE8bGEdFDRLTC+znWGyciut57BkuJ6ODhm/nA\nIaLpRPQYES0nojeJ6IveeMXeNxHVEdGLRPS6d8/f8cZnEdEL3r3dSUQ13nit932lt71tOOc/GIgo\nRkSvEtE93veKvmciWktEbxDRa0S0xBsbsv/bVacgKrwh4G8BnCGNXQXgEcbYXACPeN8B9/7nev8u\nA3DDEM2x2KQBfJUxthDAEQAu936flXzf/QBOYowdCGAxgDOI6AgA/wngZ4yxOQB2AbjU2/9SALu8\n8Z95+41UvgjgLeF7NdzziYyxxUI669D932aMVdU/AEcC+Kfw/ZsAvjnc8yri/bUBWCZ8fwfAZO/z\nZADveJ9vBHChar+R/A/A3wCcWi33DaABwCsADodbMBX3xv3/53Drio70Pse9/Wi45z6Ae53mCcST\nANwDgKrgntcCmCCNDdn/7aqzIFB9DQFbGWObvc9bAPA1RivuOXhuhIMAvIAKv2/P1fIagG0AHgKw\nCsBuxlja20W8L/+eve2dAMYP7YyLwn8D+DqArPd9PCr/nhmAB4noZSK6zBsbsv/bZVcoZykdjDFG\nRBWZtkZEjQDuBvAlxlgXEfnbKvG+GWMZAIuJqBnAXwCU78LnRYCI3gdgG2PsZSI6YbjnM4Qcwxjb\nSEQTATxERG+LG0v9f7saLYi8DQErjK1ENBkAvJ/bvPGKeQ5ElICrHP7AGPuzN1zx9w0AjLHdAB6D\n615p9vqZAcH78u/Z2z4GwI4hnupgORrAOUS0Fu46MScB+B9U9j2DMbbR+7kN7ovAYRjC/9vVqCBe\nAjDXy36oAfARAH8f5jmVkr8DuNj7fDFcHz0f/4SX+XAEgE7BbB0xkGsq/BrAW4yxnwqbKva+iajF\nsxxARPVwYy5vwVUU53m7yffMn8V5AB5lnpN6pMAY+yZjbBpjrA3u3+yjjLGPoYLvmYhGEVET/wzg\nNADLMJT/t4c7CDNMgZ+z4HaNXQXgW8M9nyLe1x0ANgNIwfU/XgrX7/oIgBUAHgYwztuX4GZzrQLw\nBoD24Z7/AO/5GLh+2qUAXvP+nVXJ9w3gAACveve8DMA13vhsAC8CWAngTwBqvfE67/tKb/vs4b6H\nQd7/CQDuqfR79u7tde/fm1xWDeX/bVtJbbFYLBYl1ehislgsFksErIKwWCwWixKrICwWi8WixCoI\ni8VisSixCsJisVgsSqyCsFQlRJTxOmTyf8auvkT0OSL6RBGuu5aIJgzguNOJ6DteJ8/7BzsPiyUK\nttWGpVrpZYwtjrozY+yXpZxMBI6FWxR2LICnh3kulirBWhAWi4D3hv8jrwf/i0Q0xxu/loj+1ft8\nJbnrTywlov/1xsYR0V+9seeJ6ABvfDwRPUjuug2/glvMxK/1ce8arxHRjV4renk+F3hN+a6E26zu\nZgCfJKJKrv63lAlWQViqlXrJxXSBsK2TMbY/gJ/DFcoyVwE4iDF2AIDPeWPfAfCqN3Y1gNu88W8D\neJoxtghuL50ZAEBECwBcAOBoz5LJAPiYfCHG2J1wO9Qu8+b0hnftcwZz85b/3979u1QZRgEc/x5z\nkSIRXByizS2QagyXJhEJLIj8ExJnCRtqbUjwDwiCxNVFcNEgoYY2IUfBZiEpaD0Oz3Pxcn2Ui2EO\nfj/L+/K8v5673HOf817OUT9MMem6Oi/FtNa1XW4c3wVWI2IdWK9jj4CnAJm5XVcOt4FJYLaOb0TE\nr3r+Y+AB8L1Wnh3ipOhar3Fgv+7fzMw/fXw+6Z8ZIKTT8oz9jmnKF/8MsBQR9y7wjAA+Zuarc08q\nbSZHgcGI2APGasppITN3LvBcqW+mmKTTnndtv3UfiIgB4E5mfgYWKWWkbwE71BRR7VdwmJm/gS/A\nXB2fAkbqrbaAZ7XOf+cdxt3eiWRpM7kBPAHeUQq2TRgc9D+4gtB1NVR/iXdsZmbnr64jEbFL6f38\noue6G8CniBimrAJWMvMoIt4AH+p1fzkpx/wWWIuIH8BX4CdAZu5FxGtKt7ABSgXeeeCgMdf7lJfU\nL4H3jePSpbCaq9SlNqR5mJmHVz0X6aqZYpIkNbmCkCQ1uYKQJDUZICRJTQYISVKTAUKS1GSAkCQ1\nGSAkSU3HA10VJGIUmCgAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RUOHh5sAgCjE",
        "colab_type": "text"
      },
      "source": [
        "# **Noisy Dueling DQN**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fshAYIRcfkNg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class NoisyLinear(nn.Linear):\n",
        "  # Noisy Linear Layer for independent Gaussian Noise\n",
        "  def __init__(self, in_features, out_features, sigma_init=0.017, bias=True):\n",
        "    super(NoisyLinear, self).__init__(in_features, out_features, bias = bias)\n",
        "    # make the sigmas trainable:\n",
        "    self.sigma_weight = nn.Parameter(torch.full((out_features, in_features), sigma_init))\n",
        "    # not trainable tensor for the nn.Module\n",
        "    self.register_buffer(\"epsilon_weight\", torch.zeros(out_features, in_features))\n",
        "    # extra parameter for the bias and register buffer for the bias parameter\n",
        "    if bias: \n",
        "      self.sigma_bias = nn.Parameter(torch.full((out_features,), sigma_init))\n",
        "      self.register_buffer(\"epsilon_bias\", torch.zeros(out_features))\n",
        "    \n",
        "    # reset parameter as initialization of the layer\n",
        "    self.reset_parameter()\n",
        "    \n",
        "  def reset_parameter(self):\n",
        "    \"\"\"\n",
        "    initialize the parameter of the layer and bias\n",
        "    \"\"\"\n",
        "    std = math.sqrt(3/self.in_features)\n",
        "    self.weight.data.uniform_(-std, std)\n",
        "    self.bias.data.uniform_(-std, std)\n",
        "    \n",
        "    \n",
        "  def forward(self, input):\n",
        "    # sample random noise in sigma weight buffer and bias buffer\n",
        "    self.epsilon_weight.normal_()\n",
        "    bias = self.bias\n",
        "    if bias is not None:\n",
        "      self.epsilon_bias.normal_()\n",
        "      bias = bias + self.sigma_bias * self.epsilon_bias\n",
        "    return F.linear(input, self.weight + self.sigma_weight * self.epsilon_weight, bias)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "class Dueling_QNetwork(nn.Module):\n",
        "    \"\"\"Actor (Policy) Model.\"\"\"\n",
        "\n",
        "    def __init__(self, state_size, action_size, seed, fc1_units=64, fc2_units=64):\n",
        "        \"\"\"Initialize parameters and build model.\n",
        "        Params\n",
        "        ======\n",
        "            state_size (int): Dimension of each state\n",
        "            action_size (int): Dimension of each action\n",
        "            seed (int): Random seed\n",
        "            fc1_units (int): Number of nodes in first hidden layer\n",
        "            fc2_units (int): Number of nodes in second hidden layer\n",
        "        \"\"\"\n",
        "        super(Dueling_QNetwork, self).__init__()\n",
        "        self.action_size = action_size\n",
        "        self.seed = torch.manual_seed(seed)\n",
        "        self.noisy_layers = [NoisyLinear(state_size,fc1_units),\n",
        "                             NoisyLinear(fc1_units,fc2_units),\n",
        "                             NoisyLinear(fc2_units,action_size),\n",
        "                             NoisyLinear(fc2_units,1)]\n",
        "        \n",
        "        self.network = nn.Sequential(self.noisy_layers[0],\n",
        "                                     nn.ReLU(),\n",
        "                                     self.noisy_layers[1],\n",
        "                                     nn.ReLU())\n",
        "        self.advantage = nn.Sequential(self.noisy_layers[2])\n",
        "        self.value = nn.Sequential(self.noisy_layers[3])\n",
        "\n",
        "    def forward(self, state):\n",
        "        x = self.network(state)\n",
        "        value = self.value(x)\n",
        "        value = value.expand(x.size(0), self.action_size)\n",
        "        advantage = self.advantage(x)\n",
        "        Q = value + advantage - advantage.mean()\n",
        "        return Q\n",
        "      \n",
        "    def noisy_layer_sigma_snr(self):\n",
        "      \"\"\"\n",
        "      function to monitor Noise SNR (signal-to-noise ratio) RMS(mu)/RMS(sigma)\n",
        "      \"\"\"\n",
        "      return [((layer.weight**2).mean().sqrt() / (layer.sigma_weight**2).mean().sqrt()).item() for layer in self.noisy_layers]\n",
        "    \n",
        "    \n",
        "    \n",
        "class Agent():\n",
        "    \"\"\"Interacts with and learns from the environment.\"\"\"\n",
        "\n",
        "    def __init__(self, state_size, action_size, seed):\n",
        "        \"\"\"Initialize an Agent object.\n",
        "        \n",
        "        Params\n",
        "        ======\n",
        "            state_size (int): dimension of each state\n",
        "            action_size (int): dimension of each action\n",
        "            seed (int): random seed\n",
        "        \"\"\"\n",
        "        self.state_size = state_size\n",
        "        self.action_size = action_size\n",
        "        self.seed = random.seed(seed)\n",
        "        \n",
        "\t    # Q-Network\n",
        "        \n",
        "        self.qnetwork_local = Dueling_QNetwork(state_size, action_size, seed).to(device)\n",
        "        self.qnetwork_target = Dueling_QNetwork(state_size, action_size, seed).to(device)\n",
        "\n",
        "\n",
        "        self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=LR)\n",
        "\n",
        "        # Replay memory\n",
        "        self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed)\n",
        "        # Initialize time step (for updating every UPDATE_EVERY steps)\n",
        "        self.t_step = 0\n",
        "    \n",
        "    def step(self, state, action, reward, next_state, done):\n",
        "        # Save experience in replay memory\n",
        "        self.memory.add(state, action, reward, next_state, done)\n",
        "        \n",
        "        # Learn every UPDATE_EVERY time steps.\n",
        "        self.t_step = (self.t_step + 1) % UPDATE_EVERY\n",
        "        if self.t_step == 0:\n",
        "            # If enough samples are available in memory, get random subset and learn\n",
        "            if len(self.memory) > BATCH_SIZE:\n",
        "                experiences = self.memory.sample()\n",
        "                self.learn(experiences, GAMMA)\n",
        "\n",
        "    def act(self, state):\n",
        "        \"\"\"Returns actions for given state as per current policy.\n",
        "        \n",
        "        Params\n",
        "        ======\n",
        "            state (array_like): current state\n",
        "            eps (float): epsilon, for epsilon-greedy action selection\n",
        "        \"\"\"\n",
        "        state = torch.from_numpy(state).float().unsqueeze(0).to(device)\n",
        "        self.qnetwork_local.eval()\n",
        "        with torch.no_grad():\n",
        "            action_values = self.qnetwork_local(state)\n",
        "        self.qnetwork_local.train()\n",
        "\n",
        "        return np.argmax(action_values.cpu().data.numpy())\n",
        "\n",
        "\n",
        "    def learn(self, experiences, gamma):\n",
        "        \"\"\"Update value parameters using given batch of experience tuples.\n",
        "        Params\n",
        "        ======\n",
        "            experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples \n",
        "            gamma (float): discount factor\n",
        "        \"\"\"\n",
        "        states, actions, rewards, next_states, dones = experiences\n",
        "\n",
        "        # Get max predicted Q values (for next states) from target model\n",
        "        Q_targets_next = self.qnetwork_target(next_states).detach().max(1)[0].unsqueeze(1)\n",
        "        # Compute Q targets for current states \n",
        "        Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))\n",
        "\n",
        "        # Get expected Q values from local model\n",
        "        Q_expected = self.qnetwork_local(states).gather(1, actions)\n",
        "\n",
        "        # Compute loss\n",
        "        loss = F.mse_loss(Q_expected, Q_targets)\n",
        "        # Minimize the loss\n",
        "        self.optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        self.optimizer.step()\n",
        "\n",
        "        # ------------------- update target network ------------------- #\n",
        "        self.soft_update(self.qnetwork_local, self.qnetwork_target, TAU)                     \n",
        "\n",
        "    def soft_update(self, local_model, target_model, tau):\n",
        "        \"\"\"Soft update model parameters.\n",
        "        θ_target = τ*θ_local + (1 - τ)*θ_target\n",
        "        Params\n",
        "        ======\n",
        "            local_model (PyTorch model): weights will be copied from\n",
        "            target_model (PyTorch model): weights will be copied to\n",
        "            tau (float): interpolation parameter \n",
        "        \"\"\"\n",
        "        for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):\n",
        "            target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)\n",
        "\n",
        "\n",
        "class ReplayBuffer:\n",
        "    \"\"\"Fixed-size buffer to store experience tuples.\"\"\"\n",
        "\n",
        "    def __init__(self, action_size, buffer_size, batch_size, seed):\n",
        "        \"\"\"Initialize a ReplayBuffer object.\n",
        "        Params\n",
        "        ======\n",
        "            action_size (int): dimension of each action\n",
        "            buffer_size (int): maximum size of buffer\n",
        "            batch_size (int): size of each training batch\n",
        "            seed (int): random seed\n",
        "        \"\"\"\n",
        "        self.action_size = action_size\n",
        "        self.memory = deque(maxlen=buffer_size)  \n",
        "        self.batch_size = batch_size\n",
        "        self.experience = namedtuple(\"Experience\", field_names=[\"state\", \"action\", \"reward\", \"next_state\", \"done\"])\n",
        "        self.seed = random.seed(seed)\n",
        "    \n",
        "    def add(self, state, action, reward, next_state, done):\n",
        "        \"\"\"Add a new experience to memory.\"\"\"\n",
        "        e = self.experience(state, action, reward, next_state, done)\n",
        "        self.memory.append(e)\n",
        "    \n",
        "    def sample(self):\n",
        "        \"\"\"Randomly sample a batch of experiences from memory.\"\"\"\n",
        "        experiences = random.sample(self.memory, k=self.batch_size)\n",
        "\n",
        "        states = torch.from_numpy(np.vstack([e.state for e in experiences if e is not None])).float().to(device)\n",
        "        actions = torch.from_numpy(np.vstack([e.action for e in experiences if e is not None])).long().to(device)\n",
        "        rewards = torch.from_numpy(np.vstack([e.reward for e in experiences if e is not None])).float().to(device)\n",
        "        next_states = torch.from_numpy(np.vstack([e.next_state for e in experiences if e is not None])).float().to(device)\n",
        "        dones = torch.from_numpy(np.vstack([e.done for e in experiences if e is not None]).astype(np.uint8)).float().to(device)\n",
        "  \n",
        "        return (states, actions, rewards, next_states, dones)\n",
        "\n",
        "    def __len__(self):\n",
        "        \"\"\"Return the current size of internal memory.\"\"\"\n",
        "        return len(self.memory)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PoOvj-62fkHm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 368
        },
        "outputId": "b06a6d74-9ffd-430f-c764-249961458ab4"
      },
      "source": [
        "def dqn(n_episodes=1000, max_t=1000):\n",
        "    \"\"\"Deep Q-Learning.\n",
        "    \n",
        "    Params\n",
        "    ======\n",
        "        n_episodes (int): maximum number of training episodes\n",
        "        max_t (int): maximum number of timesteps per episode\n",
        "\n",
        "    \"\"\"\n",
        "    scores = []                        # list containing scores from each episode\n",
        "    scores_window = deque(maxlen=100)  # last 100 scores\n",
        "\n",
        "    for i_episode in range(1, n_episodes+1):\n",
        "        state = env.reset()\n",
        "        score = 0\n",
        "        for t in range(max_t):\n",
        "            action = agent.act(state)\n",
        "            next_state, reward, done, _ = env.step(action)\n",
        "            agent.step(state, action, reward, next_state, done)\n",
        "            state = next_state\n",
        "            score += reward\n",
        "            if done:\n",
        "                break \n",
        "        scores_window.append(score)       # save most recent score\n",
        "        scores.append(score)              # save most recent score\n",
        "\n",
        "        print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)), end=\"\")\n",
        "        if i_episode % 100 == 0:\n",
        "            n1, n2, n3, n4 = agent.qnetwork_local.noisy_layer_sigma_snr()\n",
        "            print('\\rEpisode {}\\tAverage Score: {:.2f}\\tNoiseValues: {:.2f} {:.2f} {:.2f} {:.2f}'.format(i_episode, np.mean(scores_window),n1,n2,n3,n4))\n",
        "#         if np.mean(scores_window)>=200.0:\n",
        "#             print('\\nEnvironment solved in {:d} episodes!\\tAverage Score: {:.2f}'.format(i_episode-100, np.mean(scores_window)))\n",
        "#             torch.save(agent.qnetwork_local.state_dict(), 'checkpoint.pth')\n",
        "#             break\n",
        "    return scores\n",
        "\n",
        "\n",
        "env = gym.make(\"CartPole-v0\")\n",
        "action_size = env.action_space.n\n",
        "state_size = env.observation_space.shape[0]\n",
        "\n",
        "agent = Agent(state_size=state_size, action_size=action_size, seed=0)\n",
        "scores = dqn(n_episodes = 500)\n",
        "\n",
        "# plot the scores\n",
        "fig = plt.figure()\n",
        "ax = fig.add_subplot(111)\n",
        "plt.plot(np.arange(len(scores)), scores)\n",
        "plt.ylabel('Score')\n",
        "plt.xlabel('Episode #')\n",
        "plt.show()\n"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Episode 100\tAverage Score: 12.45\tNoiseValues: 19.47 5.83 7.65 7.25\n",
            "Episode 200\tAverage Score: 27.85\tNoiseValues: 18.16 4.63 7.49 8.22\n",
            "Episode 300\tAverage Score: 127.74\tNoiseValues: 24.90 6.82 13.32 12.81\n",
            "Episode 400\tAverage Score: 194.49\tNoiseValues: 23.28 8.82 19.20 15.36\n",
            "Episode 500\tAverage Score: 164.18\tNoiseValues: 28.87 9.66 23.68 19.15\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXmcHGWZ+L9P95yZzOSaSchJAoQj\nCRAkIEjAAMopoK6r4iqsqxuvFV11V0BFwXXXE13lpysIKoqICoIiIsgpymGAEEJCyEEICTkmk2My\nM5mju9/fH1XVXVVd1Vemu2eqn+/nM0nVW9db0z3v8z7nK8YYFEVRFMVPrNodUBRFUUYmKiAURVGU\nQFRAKIqiKIGogFAURVECUQGhKIqiBKICQlEURQlEBYSiKIoSiAoIRVEUJRAVEIqiKEogddXuwIHQ\n3t5uZs+eXe1uKIqijCqefvrpncaYjnznjWoBMXv2bJYtW1btbiiKoowqROSVQs5TE5OiKIoSiAoI\nRVEUJRAVEIqiKEogKiAURVGUQFRAKIqiKIGUTUCIyEwReUhEVonICyLyCbt9oojcLyJr7f8n2O0i\nIt8VkXUiskJEXleuvimKoij5KacGkQA+bYyZB5wEfExE5gGXAw8YY+YCD9j7AOcCc+2fpcAPytg3\nRVEUJQ9ly4MwxmwFttrb+0RkNTAduAhYYp/2U+Bh4LN2+83GWgP1CREZLyJT7fsoSl5+/sQr7Oju\n5y3HTqNjbCOPb+jivKOnFnz9+s4ednQPcPKhk4a9bzu6+/nl319l1sQxbOjsAWB2ewsbu/ogosv+\n1sVjTB/fzI59A0wd1wTAhs4eZkwcQ+e+AV4/ZyI7ewZZ9dpeZre3sGlXH6mUIR6LMXNiMxt39iIi\nzG4fw8udvRxxUBvnHzOVu5ZvwRjYsa+fty6cztOv7OZc+3N+ZtNuOvcN0FQf542H580DK5lXd/Wx\nrrOH04+YPKz3fXlnL1t272fx3PbA42u27aO7f4gTZk8c1ueGUZFEORGZDRwHPAlMcQ3624Ap9vZ0\n4FXXZZvtNo+AEJGlWBoGs2bNKlufldFF574BPn/nSgBe29vPpl19PPXyLp668kwmtzUVdI8zv/UI\nABu/ev6w9++u5a9x7f0vBR4TGfbHVZ1CZN5hk8eyp2+QnT2DBd2zuT7O3Clj+cQvl6fbvnbvGpIp\nw9OffxOTxjby9u//LX2sHJ+jw5JvPkwyZYb9Gad/82EgvO9nf+fRnMeHm7ILCBEZC9wOfNIY0y2u\nvwZjjBGRoqZPxpjrgesBFi1aFM2pl1I0yVTmq9A/lGTL7v0ADCZT1eqSh4FEMr191NQ25k4ey++e\ne42FM8dz58dOqWLPysNAIskRn7835zm9AwkGE5nPp7k+zsqrz+bQK+8B4MJjp/Ho2k729A0BsH8o\nycCQ9/N0PvdEqrJDQbLCz6sWZY1iEpF6LOFwizHmDrt5u4hMtY9PBXbY7VuAma7LZ9htipKXlGvK\n6h50RgpDyUz/WhvrmDa+GSBteokaDfHgoeWoqW3p7ZiIR9MY21RHPCbEY9Yksj4eY2yjdw6bjKg5\nbqRSzigmAW4EVhtjrnUd+h1wqb19KXCXq/0SO5rpJGCv+h+UQnEPGyNFa3CTSGX6NLapjmnjLcHQ\nUBfNSHMRCXy3hTPHp7djMe+A7wiD+rglIBrqJN1WZwuNRMhnq3KjPJTz23kK8D7gDBFZbv+cB3wV\neLOIrAXeZO8D3ANsANYBNwAfLWPflIhhfBqEGWEjRiLpHQib6+MARND9kKbRJyBu+eDr6WhtTO/H\nRDyan6N1OP/Xx2O02ALCua5vMGOqGwmkymRqKtd9i6WcUUyPEf79PzPgfAN8rFz9UaKNWx64TUwy\nQjzAbhNTS2Mdx8ywZtJnzz+oWl0qO86s36G1qY56V1tMhFQKWhri9A4mMbYe6GgeDfEYzukdrY1s\n3dtP32Ai8FkGU5VJQSJlaIgN/3dsKJWiMRYf9vsWSzT1W6XmcI8NQyPQxOTuU0tDnCMOamXVNWen\nwzOjiF84j22so87lmxCxfEdtzfWe8+odDaIuhthzzI6xlgbRMxCsQaSMVwgDgQLDGMPKLXsDj/UO\nJNjU1ZfvtTwBB+VyVvvfxU+YqW24UQGhRALj8kIMjEAntdsH4ThhxzSM6uVYimZsU51Hq3BMTG1N\nIQLCJUwyJqZgDSKVMuwf8gqPoLH7639aw1u+9xjPbNqddex9Nz7Jad94iD19g3T1DGQdd/JXnKgq\nKJ/TPF+gRaX8bCoglEjgMTElU4wMC26GfDPCWqC1sZ66eEZACNYgPs6nQTiKR2NdLC34220NojdU\ngzAM+ARE0Oz+Bw+vB2D/YPYA+8ymPQAsvOZ+jv+vP3uO/WVtJ2d86xHueGYze/e7BESBn+uOff30\nDxXuP8mnBVcqUk8FhBIJnD/TeEy8PojqdCeLSpkERjJN9TGPBuEM4K1NwZpUvUuYTGxpACwzUBDJ\nQA0ifPAudua/foelPSx/dY/n++XWDHNx4lce4F9vLnz1y3wCoFJasgoIJRI4g0FjXWzE50HUKiLi\n8UEM2YNrU32wM7Y+nvFBtDRa5/SECIiUMfT7kuhyCYhio4Tidr+HksaTlFeMD+Iva3cWfG4+E5I/\nYbBcqIBQIoEzFjTWxUZkHoTbZHD4lNYq9qS6eDQIW2g62sGJc7z1hRrqYrzu4AkAzJwwBsjhgzBk\naRC5Bu9EyoTeKwgn+iqZSnm0wUIyuEuJrsprYkpWJty3trxkSoRxNIg43f1D6TyDkUIiZVgwvY3/\nedsxLJjelv+CCPHdi4/jODtBzu2DGLIH12njm7n3k6dySPtYz3X18RifOetwLjx2GtPtzPMwH0Qy\nZbJs/LmsP997cC3/evMynv3Cm5lgC6hcxNOJesajDRaiQZQS6TSUyH2NX1sqF6pBKJEgrUHUj1QT\nU4q6WIyjZ4wbMbkZleKgtiZmTrQ0gLpYZshxZuIxgSMPakvnPzi/nYZ4jLp4jHnT2mhqsI71hsz6\ni/VBrNi8F4DOgGilIJyIqkTKePwOuTQIY6zcjFLqROXTgjWKSVGKwPkbbKqLk0iZnINDNUgkTWh9\noqjjzqh2m5icgTPmE5jOJ+cOc3WS5vpCNAhjyI5isr8Dsy//A5fd+mzodQ658t3SGkQq5cmKT+ZQ\nU378143MueIeT9RToeR1UqsGoSiF44RDNtZbX2nnD2ykiImhZMpjXqklnM8E8DipnYE2FjIyu2s5\niQjN9XH2hUUxBTmpXTP33z33WuB17omEW7vxU+cxMRWmQVz30DrAmzexv8BSIfl8EO5kvXKiAkKJ\nBG4nNbgExAjRJIZSxjM41hKNdRl/kFtIOqYav3xwdut9AnXi2AY69/UHPiPIxJQ0xjMTD86szmzn\nkA/piYZlYspclMgRnbar11rnwj3Yd/UWZtLSPAhFGUYyAsIajBwb7QiRDySSKU8doloizMTkOHvj\nsWATk98kN6W1KXRxoZQxWbkmKQP7+l1JbQGz/UI1CEcoDCVTRYe5unMWCtUgNA9CUYYR5w+9qT4T\nrz6SSCRNzZqYGjwCInvICXPa+8uFT25rDDwPLHOSf7BOpQz7+jMmqSBzkPuaXPLbMVclU6boMFe3\nb+R/H1jLMV/6U95rgpzQ/orFlUAFhBIp/IPKSNEghpIpj9O1FjjIXurVrTUECcm4T0BkTEw+AdGa\nWVzpgmOneY4lA6KFkj4BETTbd0ck5TIBOvdOJI3PSR38BXO3u2f7d6/YSnd//vyLoAlOIuSe5UTz\nIJRI4DcxjTSGUrUnIH78/hN4bO1Oxo/J5Bn4S4BD9sw9KIoJvBrEW46Zyu9djudUKnuwThrjMTEF\n+QsGXfkGflOXG0eDeGrjLp7auCtzz5AoJncSXpBDOZkyOZ/3mV8/x3Ov7uHLb10Q2H91UitKEaSj\nmPwaxAiJY0okTeDgGGWmtDXxD8fP8LQFCcnwKCZvu1uD8DuwU8Zk1Vcyxnhm61fe+XzWMzwaRI7P\nJ8yUFKZBuH0NQbP9QkrS/+yJV3x9yFxTqYWTyrnk6E0iskNEVrrabnOtLrdRRJbb7bNFZL/r2P+V\nq19KNEnnQfgyqEeCienHf32ZrXv7azaKyU3QrNmfByHpc72/r5aGzGfrP5Y0JquyajIF3S4N4g8r\nslcwds/K/f3w3z+IMMHhHsCDchZKWbPE3dewmlTDTTlNTD8BrgNudhqMMe9ytkXkW8Be1/nrjTEL\ny9gfJcI4DrywiJhqcvXvVwHQUKNOajf+mT+AX246n5k/LNX92fpn+6lUfh9EEG5ncK4ggmTIgB5U\n7vva+9awx5UcF2QOKiWIYsilQfzg4fUkkik+d/68ou9TDOVccvRREZkddEyssIV3AmeU6/lKbeH8\nuWXZs0eCCmGjGkT2zB+yZ+6OmdAf3ZRLQCQDsudTPh9EEO5Zud9Z7rl/yNcoSIP47oPrPPuFmpjy\nVZj1m7PcCXjlolrf2FOB7caYta62OSLyrIg8IiKnVqlfyijFGRvCyjZUir+u28nX7n0x8FilHIsj\nmSA7v18QXP++RXxkyaHMnjTG0x7PEQ2VMtmDdcoY9vQN0dpYx7sWzQzsj3ugLsRJ7aeQPIighYL6\nh5J85tfPsWbbvsy98kxm/E72QooMHijVEhAXA7e69rcCs4wxxwGfAn4hIoElL0VkqYgsE5FlnZ2d\nFeiqMjoILttQaQXin370ZHrVMj9791fGbjySCXJS+2fus9tb+Ow5R+bRILz3SZnsPIhkytDVO0h7\na2Oo+eiTty3n1V19Wff3E+ZrKGTBoCANYtOuPn7z9GYuuO4xT39z4e/D+DH1IWcOHxUXECJSB7wd\nuM1pM8YMGGO67O2ngfXA4UHXG2OuN8YsMsYs6ujoqESXlVFAKq1B+I+MHBNTdwlF26JGsJO6+Gv9\nA76VwJatQXT1DDCxpSFniPE533k0tG/uewVRbCa1Q4/tGxlMpNJm0HyOa3+m+IQx0dQg3gS8aIzZ\n7DSISIeIxO3tQ4C5wIYq9E0ZpYSamCooH/YG2IR/8teX09vdeezhtUCQkzoszNWPW2sI0iD8g3gy\nBV09g0xqaQh8rkOvHXGUFeDgul9YzaViM6kd3N8FR4DkEzZ+x/aE0axBiMitwOPAESKyWUQ+YB96\nN17zEsBpwAo77PU3wIeNMbtQlAJx/pj9ZolK6g87fIXk9vQN8iU7ggkoqexz1CgkzDX82sx2tg/C\nZJl7LBPTAJPGNhaUpOjvm3vADvMPBA3q/lycIA2ie392CZB8wubWpzZ59sdXQIMoZxTTxSHt/xzQ\ndjtwe7n6okQf50/Lb8+upAbh/vs2xvDIS14f2df+4ZjKdWaEEuiDKHCaGvdoEP7BPCCTOmXY1TtI\n+9iGvELIGJN1TtKY9AAZtu5D0KDeEI95hEKQgHBHVw0lUtCYX4PwJ85VwsSkpTaUSOCYF7LLNpRX\nQgwkknz7/rUc2tHC0TPGufoD63f0pPcvOHYaJ8yeGHSLmqKQKKYw3MLfP9sPclJ39Q6QMtaa1715\nEsuGktnXuycXYe6BoPwIv9AIil5zZ3g7+Q3FrjzX2lT+4VsDs5Vo4PggKlzOYvXWffzfI+v5j9+s\n8A0ohkGXzbhWS337CTIx5co/CLvWr4kEJco5Jr22pvq8Jqb+RDIrZ8ZjYipCg/Cbo4Iyqd0+CMe3\n4HdCh/GBxXM4tKOFjtbw6rbDhWoQSiRw/iT9Y025TUxux6j7Wf71CWq11LefIG2hcB9EuAaRDNAg\nnJLYdXHJKyAGhlJZA7t7P1SDyFNCHDImpvq4pIWBxweRzK9BuL9Lx84czxfeUt4MagfVIJRIUK0o\nJves023OSvpmtLVWyTUXE8bUp0uBQ+6V3Nx4BISIZzIQtB6EU0ajLhbLGcUElhkolYI3HTWFq+zB\n150cFxbm6h/UjUtQHdrRkr43wNjGzHzc44NI5o9i6nf5MY4/eELOdxlO9FurRIJq+SDc44bHxGS8\naxergMjw7FVn8ZElh6b3S9EgYjHxXJcy2QOso0HEY/k1iP6hFCljiEnmOe7bhSXEZS1SZO/++5sO\n5+6Pn5q+N3ijjp58OROk6ZQcDwqldSYgTjb2NRfNZ/r45pzvMpzot1aJBJlaTBXWIELaU77ErVor\n9Z0P968jV4KaG/fvMCbee/g1NnCZmGKStw7WQCJJylhrNDj39fogoLk+e62RoAKBYEVmOe/laBBt\nzcF5C3eveA0TEKbrvt89z1uVaJsqvN6J+iCUSBCWB1H+5wZvJ1PGU31TC/V5ifkG+2KvicfE/qyt\nX3pQFFNag4hLXhNT/1CKZMoKdY2lNQivk7qtuY79rqS3uphkOa8zAiJj1nJ8EONCBMT3H17PnPYW\nprjMbg6JlGHV1j1cddcLADTWV/Z7pN9aJRKEV3Mt83NdD3ALhJTxmgy01LcXt6ZXqFD3ahDi+awD\nBUQyo0E0uAR00OMGEkmMsYSQE1Xl1yDqYjE2fvX8dFs8JtkahMloECJCPCbpKKYwAQGwY98AL7zW\nDcCMCRkT0lAyxcs7e9P7/vVOyo0KCCUShK8HUbyEKKZEuPtM90Ly/sxe1SC8eExMBQoIr1Dx7gcl\nyrl9EO7ff0PAZ+FEMcWEUA3CH4lWF5Msv8HyTXvsZ8bSz06bmHLkLRhjWLllLzMnNnsS4JIpw/bu\nTIa+CghFKQHnbzmr1EYJGkQx17jPdTulkynjqZ2jYa5e3IN7oU5qtwYRF7+TOtuGn/FBeKOY/KUw\nwOWDcN3XfbukyQiy+rgwdVwTjfVxz6Rge3c/773xSbt/Vlt9TNJO6pbGXAIC1u7Yx5EHtXk0nKGk\n4eWdfen9poC+lxMVEEokyIS5+tpLuVdR57pMTD4B4Y5dry80lrNG8AiIAn81MZ+JyT2QJlMGv493\nwDEx+fIgGgNm4QOJFKmUYxay7+nTIBzt9IWrz+HR/zzdLqmR8Ul09Qymt+PxbA0iyMntZk/fEJNa\nGjyTnEQqxaZdGRNTUN/LiX5rlUgQHsVUXhOTW5o44YoAu/sG2bFvIL2vGoQXt1AoRYMQ8ZoTHQ3C\nXeF0yBXFVB9gYrrw2Gk89JklgBVGakUxZfrj9UGY9PMa6mLUx2M01vtrLmWERUbbiKVDX8c05B7c\n9+4fYlxzvWeSk0gatndnvkdN6qRWlOJx7MWF2rNzUZwGkcGtQVx43V/TTkfQPAg/bqFQaJhr3CMg\nfCYmO1HumBnjufvjiwF4pSuzEFCQiamlsS7tFxhIZKKYnOcY43VS+/vZEI/R1TOYLvPe7yqp4Qgz\n98RgjM/E5PTTef5AIkVbcz1LDp+cbk+kDDvcPogKh7nqt1aJBBkfhK+9zM91OzJzLfhS6CBYK3h9\nEIVd4/8devIgjLGqr8aEyW1WjaI1263lPC0fRLaJqS4m6W1Lg/Am4IWZmDL3ifHYup0ce819APQN\nZspnOOYw9+y/xaVBzJva5olq2rPfMk+1Ndfz8TMO43PnHQXAq7v6PIX9Ku2k1jwIJSI4mdQjw0l9\nIPesBUpxUvu1Q/E4qS1zTNwVppq+Lia458LN9Rn/gOP0HXBlUrtNTLt7B23zlckWEL7ZfI+rYmyQ\nwnhIx1gALjn5YC4/90j2uQb+tdutyr/jmuuJxYSZE61Q10tuespzj0qbmFRAKJEg7aTO+vspwQdR\nxDWeMNeQVceKvWctEC/BB+Gv1OvJg7BNTPGYZA3kdTGvQ9s57mRY18WEvqFMFJNz/G/ruvjKPavT\n1y3y1UByh8sOJVM+AZE9kC+Y1sb9/34ah3SMJe6KboJM6Q1Hq/CvmOegYa6KUgKZNakrrUG4TEwB\nC8McSD+ijJTgg/DjzYOwTEyZDOsMcV+inHNd3PYPTBrbwM59AyRTxhPFtL6zx3Mf/8I/7qzmbXv7\n0+tMQ7AvLB4T5k5pzQiogMAFR0DEfcecQn9BIbrlpJxLjt4kIjtEZKWr7UsiskVElts/57mOXSEi\n60RkjYicXa5+KdHEmaFXeslR9/3DCrpVoh+jDfcAWmpcQVCxvrogDSIu6cG4qT7m0SAAJrc2sWPf\nAMbgETB+gdDpikoDrwaxZc9+nwaR/VL+72ZQwp4jINxh0ZedOZcnrjyTP37i1IqXkimnOPoJcE5A\n+7eNMQvtn3sARGQe1lrV8+1rvi8ildWllFFNaB5EmTUIPD6IHCYmVSE8lBLmmuse1vobxlMqwyEe\nk7TJZvFhHenB2zEDTW5tZIetQcQkI7y6ei3HseMw3tU76LmvOydhy+5gAfHgp98Y2v+gyLbxjonJ\npUGcf/RUxjbWcdTUttB7lYuyCQhjzKPArrwnWlwE/NIYM2CMeRlYB5xYrr4p0WNY8yCK8kFkzg1a\ne9ghlVIB4aYsJiZbg/Cb7+tiMTpaG/nFB1/Pdy9e6Gq3NYi2JnZ091ulNlwaSFfPAK2NdSw5ogPI\n1HZycGsAr3T1ekxMzr0dx3QQ/vf+4gXzmNBildlwh+VOrsDKcWFUwwfxbyKywjZBOV6f6cCrrnM2\n222KUhCOIMgSECXdq7Rzc0YxldCPKDPcJiYnzDUei2V9B5zZ+BsOa2dMQ10mZ8YeoDtaG+nqHWQw\nkbKLADoCYpDxLfVMDqiyCt5BfMPO3vQSp1Da0rfnHz01ve12co8fE17kr9xUWkD8ADgUWAhsBb5V\n7A1EZKmILBORZZ2dncPdP2WUMpwmplKeC7md1LkqedYinkS5EiWE+zJnJbe6ABOTfy2ObXutxLOp\n46yB353hHHdVid3W3U9LQ11okT13DsyGzl7W7sg4tUtZ/8O9oJA3a7x6OTQVFRDGmO3GmKQxJgXc\nQMaMtAWY6Tp1ht0WdI/rjTGLjDGLOjo6ytthZdRgwvIgSgpzLe3cIA3irHlT+MY7juGtC1UhduMe\nP0v2QfhMTImklczmn737TTmbdlkZ1nParSVBLz5xVuaePif3uh09oQO083HXx4VVW7s9ZblLeaeG\ngAilNx01OeDMylHRPAgRmWqM2Wrvvg1wIpx+B/xCRK4FpgFzgacCbqEogYTmQZTkpC7CB+E6NygP\nQgT+cdHMrPZax1N4r2QfRGY7mbIimYL8Gf6cAieYwBEQ45rrOXfBQfxx5TZPuW/IOJJ/8cHXe2b4\nkPnsD5vcyuqt3Z5j7n489JklvLzTGzKbj/nT2vivty7gbcdVd2JRNgEhIrcCS4B2EdkMfBFYIiIL\nsf5sNwIfAjDGvCAivwJWAQngY8aYZNB9FSWIsDyIUihGpqTy+CByLURfy5RSaiPXPZwlO4NMO2H3\ndw/4jiCIi9dEdftH3gBY/gs/TimOgyeOySkg5rS3pIVRGD9+/wmefRHhvScdnPOaSlA2AWGMuTig\n+cYc538F+Eq5+qNEm2o5qd1PUAFROKWsSe1H/E7qlAnURvwmot/92ym8tqff0+aYd9y1mADmTQsP\nLXU+WvcKcA6FvtP/vfd4ZkxoZsH0cQWdX2k0k1qJBOFhrgdws0JOzadBqHwIxL+2Qym40wjcYa5g\nhYyGccyM8Zyz4CBPm6NBWGtSF/Z8x0k9c+KYrGOFOqnPWXDQiBUOoAJCiQqhCwZVsBZTIvs6/6L2\nikUpxfpy3SNdjdVuK9Y845SwiEnhs38nt2XS2IasY8Nh6hwJqIBQIkFoqY0qV3NVE1Mw8WHwQbg/\nayfr2TEVFRtm6s5pKDTs1hEk9fEY11w0n9uWnpQ+FpUForSaqxIJUqEaRHkJW3LUQQVEMO4xuHQN\nIrPtLPfpaALF5g44gmUomUpfG1Qryc0XL5hP+9hGzjhyctpEJWJNGqKiQaiAUCJBJsx1OEptFP9c\nUAFRDN41qQ/cxNTVYxXSK7UctjPADyYyCwPlW3uho7WRL10439MmWN+fUhLlRiJqYlIiQSZRzt9e\nwr2KyYNwbfvzIOa0t3DtOxeiZDMcK+y5b9E7aEXFl1oO29EgBpMmbWIqRdikS4mrgFCUkUNmydED\nlxDFaRDh60Fc+85jmZ0n/r1WGY7xU8i+SakaRINLg3DyG5obShcQpWpFIw0VEEokcAZqv4OxpCim\nEq1C/vUg/EtSKhmGpb6QfQu3r+BANYihZIr9tjbSXIKwcV5LTUyKMoIYzjyIYoSKu2Cb393QWOH1\ng0cTzu+tNaQQXjG0NWfu0TgMPohDJ7dw/MET+Mrbji76Po5pSZ3UijKCyJiYhuNmxT8XvMICvFVC\nFS8zJ4zhdbPGc6W9GE+hfOiNh7DPte4CQGtTPTvtKKamEjUIR0AMJVM01sXTJTaKxfn6RUWDUAGh\nRILQUhtl90Fktv2LAo2p1z+vMJob4tzx0VOKvu6Kc10Cxf51u8txl65BWN+bgRxrehRC2gcREQ1C\ndWAlEjhjsz96pJJrUvtNTKU4OZXiaXOttZEvNDUMx3eRa02PQnDkQim+r5GICgglEjh/jllBTKXk\nQRRlYnL7ILwX1kckm3ak416MqdTAgOMPngjAP79h9gH15fPnzyMm0NIYDe0xGm+h1DwZE5OvvZR7\nlViLyW9iquZKYLWAs8Tn9PGZaqpuDeLUue0MFqgRdLQ2svGr5x9wn955wkzeeUJ01v9QAaFEikrX\nYnJLiGSp8bFKSazZvg+AUw5r54ePbgC8GsTPPvD6qvQrSqiJSYkEqZA8iFJ0iOLkQ3iYq1IZTpwz\nMb1dah6EEoz+NpVIkK7FNCwaRDFLjma2/SYmpby86ajJ1MXEkz2tAmJ4KeeSozcBbwF2GGMW2G3f\nAC4ABoH1wPuNMXtEZDawGlhjX/6EMebD5eqbEj1CndSl3KsYJ7Vr2++kVsrLjy49IUso1+WpwKoU\nRzl/mz8BzvG13Q8sMMYcA7wEXOE6tt4Ys9D+UeGgFEV4NdfyPtctFLRya+WJSs2jkUrZBIQx5lFg\nl6/tPmOMkwb5BDCjXM9XaoswH0S549HdAkgVCCVqVFMf+xfgj679OSLyrIg8IiKnhl0kIktFZJmI\nLOvs7Cx/L5VRRVaYa7lXlHNtq4mpehw1ta3aXYgkVQlzFZHPAQngFrtpKzDLGNMlIscDd4rIfGNM\nt/9aY8z1wPUAixYt0r9IBcg4locj96AorcNtYlIBUTV++9E3MHCAWdBKNhXXIETkn7Gc1/9k7L9q\nY8yAMabL3n4ay4F9eKX7poyhnCw1AAAgAElEQVReMlFMvvYDuFdB57q23S6IY2eOL+HJSqk01cc9\nGdXK8FBRDUJEzgH+E3ijMabP1d4B7DLGJEXkEGAusKGSfVNGN6G1mCq05Gg8JumImts/cjLzpo4r\n+rmKMtIoZ5jrrcASoF1ENgNfxIpaagTut00BTjjracA1IjIEpIAPG2N2Bd5YUQJwzEJBq4wVfa+i\n8iAyJT4StoCYOq5ZC/UpkaBsAsIYc3FA840h594O3F6uvijRJ70eRCy4vah7lXBuTMTlByn+mYoy\nEtGsEiUSpBPlstorE+bqzuAeDi1GUUYCKiCUSGCMQaTyxfqcU92+D9UglKigAkKJBMZY2kOWBlGS\nAlGaD8JB5YMSFVRAKJHAYBCRytdiCirxoRJCiQgFCwgRWSwi77e3O0RkTvm6pSjFYYw1i/fb/8se\n5kr2Wtjqg1CiQkECQkS+CHyWTHG9euDn5eqUohSLwRqYq6ZBiPoglOhRqAbxNuBCoBfAGPMa0Fqu\nTilKsaQcJ8QwUMqSo2phUqJIoQJi0C6LYQBEpKV8XVKUEnCc1Ae+oNwwaBAqIpRoUKiA+JWI/BAY\nLyL/CvwZuKF83VKU4jBYwiHLB1HuPAj7/u4wV12iQIkKBWVSG2O+KSJvBrqBI4CrjDH3l7VnilIE\nxhhiQVFMFdIgxGNiUgmhRIO8AkJE4sCfjTGnY60IpygjjlSIC6IkJ3UJV8VEnRBK9MhrYjLGJIGU\niGh5SmXEYoxl+x+ORLniNIhsE5O6IJSoUGixvh7geRG5HzuSCcAYc1lZeqUoRWIwtpO6sj6IVKCJ\nSVGiQaEC4g77R1FGJJYGMTylNkrxQcQ1ikmJIIU6qX8qIg1kVnlbY4wZKl+3FKU4rGJ9w5QoV1Qe\nRICJqYRnKspIpCABISJLgJ8CG7G+/zNF5FJjzKPl65qiFE46zHUYwphKi2JSH4QSPQo1MX0LOMsY\nswZARA4HbgWOL1fHFKUYwhKpy+uBCMukVgmhRINCE+XqHeEAYIx5CaseU05E5CYR2SEiK11tE0Xk\nfhFZa/8/wW4XEfmuiKwTkRUi8rpiX0apXQwmHWrqnsGXe0U55wEeJ7XKByUiFCoglonIj0Rkif1z\nA7CsgOt+Apzja7sceMAYMxd4wN4HOBeYa/8sBX5QYN+UGuVv63fyxIYuwM6DsAfmAx2f3RVgH1/f\nxePru8LPJTuDWwWEEhUKFRAfAVYBl9k/q+y2nNg+il2+5ouw/BnY/7/V1X6zsXgCq6zH1AL7p9Qg\n77nhSd59/ROAM5F3NIjMCH2g5b4vvuEJLr7hifBzbdOWmpiUKFKoD6IO+F9jzLWQzq5uLPGZU4wx\nW+3tbcAUe3s68KrrvM1221ZXGyKyFEvDYNasWSV2QYkeJlCDKHu5b3uhItRJrUSQQjWIB4Bm134z\nVsG+A8JdIbaIa643xiwyxizq6Og40C4oEcFZMAgO3AdR3JKj2UudqnxQokKhAqLJGNPj7NjbY0p8\n5nbHdGT/v8Nu3wLMdJ03w25TlLykjAk07ZRfg7DqMHlMTKpCKBGhUAHR644qEpFFwP4Sn/k74FJ7\n+1LgLlf7JXY000nAXpcpSlFyYjxO6gPzQRSDs1CRJw+irE9UlMpRqA/ik8CvReQ1e38q8K58F4nI\nrcASoF1ENgNfBL6Ktb7EB4BXgHfap98DnAesA/qA9xfYN0Wxlxy1OcARujibZ4CJSSWEEhFyCggR\nOQF41RjzdxE5EvgQ8HbgXuDlfDc3xlwccujMgHMN8LG8PVYUH8mU4fH1XelZvMdJXe5aTFgCQVeU\nU6JIPhPTD4FBe/tk4Erg/wG7gevL2C9FKZjvPbiWLXv2s2WPZfX0OKlL8EIUY5Yyju9DZYISQfKZ\nmOLGGCeP4V3A9caY24HbRWR5ebumKIXx943eVBuvD6L4+xVzib+KrCoPSpTIp0HERcQRImcCD7qO\nFeq/UJSy0tUz6Nl3D9L7+hNs3VtcPEXRJiYyJqaYSgglQuQTELcCj4jIXVhRS38BEJHDgL1l7pui\nFMSuXp+AcG1f99A6Tv6fBymGosp9OyvZDVOZD0UZSeTUAowxXxGRB7Cilu4zGeNsDPh4uTunKIWQ\nJSBKmMV7/A7FZlLjCrFVCaFEiLxmIrsukr/tpfJ0R1GKJ5HyjuiljNGlpksY28aUriSrOoQSIQpN\nlFOU0UMJY7QJ2S4Ej99B5YMSIVRAKApeE1Mx2kTKGM9KdioflCihAkIZ9TTUeb/GgSvL5Rn1vRpE\n8cX6ggoFKspoRwWEMipxD/hx36gc5KQeShY36Bd8rl3uO50HoTqEEiFUQCijkqTLMZ3yjehBs/ih\nZCrn/YKCmFKuZ8y+/A+h11lRTNnLnSrKaEcFhDIqSZocAiLg/EQeDcJtVnK0E390VPB1Ti2m8Gcr\nymhFBYQyKnFrEFlhrgHT+MESNIik776pAIGRWeo0e7lTRRntqIBQRiVuoeD3GQRqEKncAiL4Gd5r\nhgLvYbwahMoHJUKogFBGJUGzeYdAH0Qij4kpIBHCLw+CzFQZH4T97JxPUZTRhQoIZVSS2z+QPUxv\n3bufIz7/R57dtDvwCo8PAscH4ZUQoQJCMtFLamJSokTFBYSIHCEiy10/3SLySRH5kohscbWfV+m+\nKaMHv3/ATdAY/Ze1OxlIpPjRY3nXuUprE/5nBJmYDMZakzoW/mxFGa1UvGS3MWYNsBBAROLAFuC3\nWEuMftsY881K90kZffg1iFPntvMvi+cAwWaeosJcTfAzgu6RSi85qpnUSvSotonpTGC9MeaVKvdD\nGWX4fRAnHTKJ04+YDATP4vNGMQVs+zWIcBNTZsUgNTEpUaLaAuLdWGtOOPybiKwQkZtEZEK1OqWM\nfLJDW/Ocny8PwlOLydrOMjEFCBnHXxHTWkxKBKmagBCRBuBC4Nd20w+AQ7HMT1uBb4Vct1RElonI\nss7Ozor0VRl5JH3+AHdF1aByF3lNTAFtfiEU6BjXJUeVCFNNDeJc4BljzHYAY8x2Y0zSGJMCbgBO\nDLrIGHO9MWaRMWZRR0dHBburjCT8g3XMNTCXZGIqIFHOLWSMMVzz+1Ws2tptV3NNPz1PzxVl9FBN\nAXExLvOSiEx1HXsbsLLiPVJGDf7B26tBZJPPxOQm46QOD3Pd3TfETX99mRe37UOQjIlJ5YMSISoe\nxQQgIi3Am4EPuZq/LiILsSZwG33HFMVD7jDX4k1MXhtTsA/CLTD8GosEtCvKaKcqAsIY0wtM8rW9\nrxp9UUYn2Sam3CNzfh9EdumObBNTcHkPSf+j5b6VaFHtKCZFKYlsE1NmO7jcd+GlNgoJc3VXkxVR\nE5MSTVRAKKOSLAHhkhAlrQfh3g5LlHOZmNzP90Qx5XyKoowuVEAooxK/gJADDXN150GE+SDcGoRb\nQOAq1qcqhBIhVEAoo5Jiw1yLWXLUIVtAhGkQktcHoiijERUQyqgkV6Jc0JrSpZiYsov1FaJB5HyM\noowqVEAooxL/eO/WIExAXvRgovhEuaxMarcGYbw+iMyKcjkfoyijChUQyqjEr0Hks/33J5I5jwet\nSe1/htsHkfJoEOJak1olhBIdVEAoo5KsYn15zu8dyC0ggoox5YpiSvijmNTEpEQQFRDKqCRXqY0g\negYS1kaIr7oQH0RYFBNkNAd1VitRQgWEMirJzoPIfX5xPgh7yVFf5NNQMsUzm3bTP5QklZUoZ2/n\n6beijCZUQCijkmJLbRRDmAaxdnsPb//+3/jve1YHRDGphFCihwoIZVSSK1EuJyGnBUU++degfnH7\nPgBe3dWXlUmd5/aKMipRAaGMSnIlypVC0JrUfiG0ZXcfAJNbmzzHYp5aTCoilOigAkIZlfjXpD5Q\nE1PQmtT+7OudPYMATGlrzMqDUAuTEkVUQCijklwaRFAmdT6C16QOdmw31sezfRDOtkoIJUKogFBG\nHYOJFHv3D3naJE+pjXwEZVKH1W9KJI3X/CSSriariXJKlKjKgkGKciC898YneerlXZ42t4kp12pz\nBeGU+w4REMlUyhvmimoQSjSpmoAQkY3APiAJJIwxi0RkInAbMBtr2dF3GmN2V6uPysjELxzAa/tP\nlqJCuMiU+w42MfUnUgwmfFFMKhiUCFJtDeJ0Y8xO1/7lwAPGmK+KyOX2/mer0zVlNOFOlDMlCIig\nKKahEE3k+kc3ePaFjAajmdRKlBhpPoiLgJ/a2z8F3lrFviijCLcPwj+uN9W7pUfw9UF5EIWaqkRE\nTUxKJKmmgDDAfSLytIgstdumGGO22tvbgCnV6Zoy2sjlg2hpqAs95hDspM5dnsNB14NQoko1TUyL\njTFbRGQycL+IvOg+aIwxIpL112wLk6UAs2bNqkxPlRGJSGZgd4e5pnwmppbGOrp6rRyGMP9E4JrU\nBa5ClzQmkyinzgglQlRNgzDGbLH/3wH8FjgR2C4iUwHs/3cEXHe9MWaRMWZRR0dHJbusjDDirum6\nW4PwJ9E118dDjwWRLtZXoIkpmTJqYlIiSVUEhIi0iEirsw2cBawEfgdcap92KXBXNfqnjA7cQkE8\nGoT3vKaGjIAI1SA8iXLW/4kCTUxDSZPugMoHJUpUy8Q0Bfit7VisA35hjLlXRP4O/EpEPgC8Aryz\nSv1TRgOu0djjgzB+DSIzDwr1Qbi2P3/nSlZs3lOwk3owkTzgWlCKMhKpioAwxmwAjg1o7wLOrHyP\nlFGJa/yOeTKpc5iYQjUI7/6vlm3mgmOnFdSN/qGU+h6USDLSwlwrhjGGi657jHue35r/ZGXE4zYx\n+Wf+zW4TU6hWkN1eqIlpIJFS34MSSWpWQOzdP8Rzm/fyqV8tr3ZXlBJx5y7EcvkgXBrEExt28dtn\nN2ffq4A1qcMYGMqYmA6wyIeijChqUkA8v3kvC6+5H4ApbU1V7o1SKu5BPdc6DG4BAfDvtz2X3k4k\nUyy9eRnLXsmu6FKcBqEqhBI9ql1qoyq0tzakt6e0qoCIArlKXDT7BISbbd393LdqO/et2p51rFAN\nYtAlSFRMKFGiJjWIg1xaQ0dbYxV7ohwI7uE7VxSRp9SGjz19Q6HHCk2Us54vWX1SlNFOTWoQbnPA\nuOb6KvZEORDc0UrFahBX3LECgLam8M8/EVLNNQi1MClRpCYFBMCP338C7//x3wvKrFVGJu5PLtcA\n7fdBANz61Kt5759IGWKS7fQOIu2k1q+TEiFq0sQEcPoRk5k+vjl01TBldJFTg2gI90HkIpE01MUK\n+xPRPAglitSsgACoi0voojDKyMeEJMr5aaorTUA8v2Uv8QJTpNXEpESRmhYQ8ZgUHKmijGxyO6lL\nExAAdTlu3BDP/PlI2kmt3yclOtS0gKiLSVGRKsrIJdcMvlAtIIhCJxCqQChRpMYFREw1iIiQK1Et\nn3zIpSXsH0qGHnPXdXIeX6jPQlFGAzX9bVYfRHQI8kG0j7USIvNlOU8dX1qyZDIgzLa1qWYDA5UI\nUtMCQn0Q0SFICTh6+jgANu3qy3nttHHNee8/fkx2voS31If1f5vm1SgRoqYFRH0spj6IUUZY3kqQ\nBvEvi+cAsPiw9pz3nD4+XEA4guc771rIxq+eT5tPQzh6+jje/rrp6f1ciXeKMtqoaX04HpOCF4VR\nRgZhGl+QFenIg9rY+NXzs9pPOWwSf13Xld7PZWJqrIuz+svnpPf9T//9xxcD8P2H1wFkCRBFGc3U\ntAZRFxeG1AcxqggT6EEaRJjz+ZYPnuTZbx8bXo8ry0kdMp/Y158A1MSkRIuKCwgRmSkiD4nIKhF5\nQUQ+Ybd/SUS2iMhy++e8cvelTjWIUUeYQA8SEDGXgJg6LlxLmDCmIfTY0tMO8eyHfVu691tF/1SD\nUKJENb7NCeDTxphnRKQVeFpE7rePfdsY881KdSSuPohRRzLk8wpSFtwaxONXhK9kG+SABrjw2Glc\ned5RnrawJUsdDaJVfRBKhKi4BmGM2WqMecbe3gesBqbnvqo81MelqIqdSnUwxvDMpt0YY9jY1Rt8\nUoCAKDRBbk57S2D7uQsOCuhLZnvRwRPS2/v6bQ2iWTUIJTpU9dssIrOB44AngVOAfxORS4BlWFpG\n1jJfIrIUWAowa9asA3q+hrmOTPoGE2zvHkgP3Hc8s4VP//o5fvBPr+MjtzwTeE2giamAAkkrvnQW\nbU31NNTFGEykuO49x7H4sHaa6uOBJTrcpTQ+9MZD09tLjpjMQ2s6OXxKa95nKspooWpOahEZC9wO\nfNIY0w38ADgUWAhsBb4VdJ0x5npjzCJjzKKOjo4D6oP6IEYmS29+mtO/+XA6pPXvG3cB0NkzEHpN\nMU5qgLcdZymtrY3WHKnDdlS3NdUzfkxDaP0m57pnvvBm3jxvSrr9kpMPZuXVZzNjwpjQZyrKaKMq\nAkJE6rGEwy3GmDsAjDHbjTFJY0wKuAE4sdz9qIurD2Ik8ti6nQDsG7Ds+jt7BoHcJqOgQ7Ec53/j\nHcfw3BfPSmdZTy5wZcEvX7SA5646i4ktXse2iDC2Uc1LSrSoRhSTADcCq40x17rap7pOexuwstx9\nqYupD2Iks6fPEgw7bc3h5c4Q/wP5y2n4qYvHPKsJTm61BESPLZRyXhfi1FaUqFGNKc8pwPuA50Vk\nud12JXCxiCzEiiTcCHyo3B3RRLmRze6+IQ6eBDu6+wFY19kTeu4BFGwFoMMWEJ37ws1YilJrVFxA\nGGMeI7g68j2V7kt9PBapFeWSKcNAIsmYhtI+1n39Q7Q21dM/lCQmQkNddVxUdXbwwO6+Qfb2DfHa\nXltA7PAKiBNnT+Qp2z9RiEM6FyfOmcTPn9iU1iQURanxTOpKahC9eUwXYfQPJTEFLnT8H79+jnlX\n/amo+yeSKYaSKVZs3sPCa+5nfWcPl9z4FNfc/UIp3Q0l6D36Q0ppO76GPX2DrHxtb7p98+79APzk\n/Sfw0GeW8OP3n5A+dqAC4sJjp/GHyxZzTkBoq6LUKjUtICrlg9jU1cf8L/6Jn/z15aKu27a3nyO/\ncC+3PvVq1jFjDImkt+93PLsFCB94gzjr249y/JfvZ0NnL8mUYc22fWzs6uW5VzMDszGmJEFqjCGV\nMmzvtt7jF09tSret3trNkV+4l/te2JZ1nSMgdvcOsXprN5Aph9EQj3Ha3A7mtLfQ4nIKD8eSn/On\njSval6EoUaa2BUS8MivK3bfKGgS/9PtVRV23wba537l8S9axW57cxGGf+yO7egezju21yz4AebWP\nDTt76e5PsNt2CL+2Zz/d/UO8vLM3fe33H17PWd9+pGBNxuHzd67kkCvv4cVt+wC4a/lrXHLTUxxz\n9X2stc1FP3x0Q9Z1zhC9p2+Q7d39NNXHmDXRqrg6dXxTYHTSgWoQiqJkU9MCIm6vKFfswFcIr//v\nP/O+G58E4NG1O9PtxczuBxKWhhAPGPx+/sQrAKwPcNy+/r8fYPXWbv62fidzrriHNfYAnYvdfZZQ\n2djVS/9Qip6BRDrvYMXmPazv7GWb7SwulFue3ATApTc9BcDYxjr+snYnPQMJeuzSFM9v2eu5JpUy\n9A5av6Ou3kG6egaZ1NKYLmERtnaDygdFGX5qOnDbSaRKGYgP8wCzvXuA7d3WALvZtWBNV+9gzvUH\n3DjaQVD8f6OdyPWRnz/NQCKVlcH7rh8+Trc9CJ/9nUe55qL5XHLy7NBnbbJLWLiFycudvUxubWLL\nHsv2f/L/PMjr50zktg+dHHiPZMogwBMvd3HZrcuzjrtn+S/YvoXBRIrZl/+BZZ9/E+1jG1n6s6cz\nfdrVR0yESWMb0iGpHSFOZNUgFGX4qWkNos6WCkPJ/H6IR17q5Nir72NTV+7VySBbS9ixb4AZEyyh\n0JUjG9hPV6917mPrdjL78j/woZ8tSx9zZMbOnkH29Sd4+hVvVRJHODhcddcLfOHO8NSSNdstTeTF\nrS4BsdMSGq/tyWgOT768K/D6+17YxtzP3cORV93LL57clM5dcLOte39629EuHN75w8dZ/LUH+fPq\n7em2v6zdySMvdTKppYG3HjcNgOaQDOcDDXNVFCUb1SDIrDHw3/esZnfvIN/4x2Ozzv3j81vZu3+I\na+9fw3fefVzO+zozboDFX3uQnoEEbzh0Ept376crwGfg8Nd1O/nEL5dz/7+fxoSWBrp6vOfet2o7\nO3sGaB/byI7u4uP1f/bEK5w1fwpLb36aoWSKwyaPTR97abslGPa5oq1e3tlL32Aiy89xxrce5rS5\nHfz8iVeY097Crz98Mg+/1EljXZz9Q0nuXrE169kzJzazckt3aN82uJLg3v666bQ21vHTxy0z2vgx\nDZxx5BSue89xnDhnYuD16lxWlOGnxgWEpUBd/+gGHt/QRVfPAJt29XHVBfOyyjY7jt9l9ky9u3+I\nC7/3GBu7+rjqLfPSy1t+/d4Xudke2CATmnnU1DbuW7Wdj/78GRrrY/QNJjmorYmffeBEvnz3auZN\na+OJ9V3s7BnggRd38I7jZ2QJE2Pg4TWdbNm93yOEwjj+4AlZmsX7bnyKhniMs+cfxB+ezwzkQVFK\nL+/szco9AGsw39DZy4Lpbax6rZtr73+JlVv2snDmeDbt6gvs27EzxvPqLqv9hNkT+PtGq1/fu/g4\nHnmpk988vTl97tf/4RhufCwT8eU40N9yzLS876woyvBR2wLCNjF978G1uMfHx9bu5Nyjrcofn/rV\ncqaNa06bWzbv3s9F1z3GvGnj2Gibm665exX/sngO63b08P2H1wc+66ipbYC1QpmzStmmXX288RsP\nA/Dn1ds5YbZVPvreldu4/enNLHsl25zzmV8/B8CC6W2ccmg7P3x0AzMnNhMT4RWf+ctdL+jqC+fz\nm6c38/yWvYxpjHP1RfM9AgJIVzQFmDGhmftWbee+Vdtpbapj4czx/MXlbAf42JLDeGJDFzfbDvMP\nLp7DuOZ6tuzZ77kXwGfPOZLjZk3AGMOlb5jN3M/9EYALjp3GWfOncORBrdTHY7zh0EnUxWNccvJs\nXtrew+3PbE4L8kL5w2WLA6O7FEUpjpoWEI7zt6EuRv9QZjD7yC3PcEh7C//77uO44xkrxFQEjjyo\nlRe37eO5zXt5brPlZG1piNM7mORTty0PXVcAwtccAMv88uqu/elZtWOHP/2IDh5a05l1/rkLDuLr\n7ziGxro4E1oaeO9JB/PA6u184pcZx/AXL5jHW46ZxsdOP4wd3f2cNf8glhzRwRu/8TATWxo8y2xe\ned6RbNtr+UmuudsKxf3sOUfy6V8/x2AixafefDhvP24Gty3bhDFWLZTu/UOccdRkFs9tZ1xzPd39\nCd570sE8+OIO7n1hG3Mnj+WtC6fzlXtW2+84hg/YWhbAjy5ZxKSxlgBrrIvzwVO9K7c1N8T55j8e\nw7Ezx3HO/OKS1+ZPG1fU+YqiBFPTAqLenpl+/Iy53PrUprQ5CKz8gAuuewyAU+e2M3PiGN5z4ize\n8r3H0uc01MV49D9P56O3PMMdz26hqT7GsTPGcd7RU+kdTPLdB9Zy3KzxnH7EZOZOHstX3340g8kU\nV91lZSn/x9lHMHVcE+cumMo3/rSG/UNJLj5xJh/5+TNs2bOfr73jGPb1J3hw9Q72DyU5Z8FBPLB6\nBx867ZB0LsCH7TUJ3Gsh33PZqcybZmks7qifgye1cPWF8znlsEkATB/fzJY9+1kwfRxLT2tny579\naQFx6tx2Hv7MEu5e8RrvO+lg6uIxlp6WWf/AobEuzqfOOiK9f8aRk7n696tIJA3/etohTGhpYMH0\ntqzr3uQqlR2GiOSMvFIUpbzUtIA47fAOLjvjMD546hzecfwMfvK3jZy3YCqPvLSDb973EgDvPWkW\nX75oQdoJ+psPn8wnfrmcLXv2M7m1kUljG7nlg6/nuw+s5dXd+/mH181g8dx27rSzmudPa+OyM+cC\n8O4TZ5FKGXb3DvGPi2YwzRXuetUF89Lb3714Ictf3cvk1iYmt8KhHRlnctiCNKfN7eATZ87l0jfM\nzipF7ebSN8xObx81tdXjL5g2romPLjmUupgwrtlaFyFIKOTi4EktfP78ozh1rrVWxzuOn1HU9cXy\np0+eluVnURRleJByJIlVikWLFplly5blP7EEHli9ndf29vO+kw7OOvbanv18/d4XWXLEZN56XPBq\nqQOJJN+67yU+tuSwEVseemfPADc8uoHPnH0E9fGajnhWlJpCRJ42xizKe54KCEVRlNqiUAGh00ZF\nURQlEBUQiqIoSiAjTkCIyDkiskZE1onI5dXuj6IoSq0yogSEiMSB/wecC8zDWoZ0Xu6rFEVRlHIw\nogQEcCKwzhizwRgzCPwSuKjKfVIURalJRpqAmA64l0/bbLcpiqIoFWakCYi8iMhSEVkmIss6O7PL\nUCiKoijDw0gTEFuAma79GXZbGmPM9caYRcaYRR0dHRXtnKIoSi0xohLlRKQOeAk4E0sw/B14jzHm\nhZDzO4FXgo4VSDuwM+9Z0ULfuTbQd64NSn3ng40xeWfYI6oWkzEmISL/BvwJiAM3hQkH+/wDUiFE\nZFkh2YRRQt+5NtB3rg3K/c4jSkAAGGPuAe6pdj8URVFqnZHmg1AURVFGCLUuIK6vdgeqgL5zbaDv\nXBuU9Z1HlJNaURRFGTnUugahKIqihFCTAiKqBQFF5CYR2SEiK11tE0XkfhFZa/8/wW4XEfmu/TtY\nISKvq17PS0dEZorIQyKySkReEJFP2O2RfW8RaRKRp0TkOfudr7bb54jIk/a73SYiDXZ7o72/zj4+\nu5r9PxBEJC4iz4rI3fZ+pN9ZRDaKyPMislxEltltFftu15yAiHhBwJ8A5/jaLgceMMbMBR6w98F6\n/7n2z1LgBxXq43CTAD5tjJkHnAR8zP48o/zeA8AZxphjgYXAOSJyEvA14NvGmMOA3cAH7PM/AOy2\n279tnzda+QSw2rVfC+98ujFmoSuctXLfbWNMTf0AJwN/cu1fAVxR7X4N4/vNBla69tcAU+3tqcAa\ne/uHwMVB543mH+Au4KAU2A0AAATNSURBVM218t7AGOAZ4PVYCVN1dnv6e46VV3SyvV1nnyfV7nsJ\n7zrDHhDPAO4GpAbeeSPQ7mur2He75jQIaq8g4BRjzFZ7exswxd6O3O/BNiMcBzxJxN/bNrUsB3YA\n9wPrgT3GmIR9ivu90u9sH98LTKpsj4eF7wD/CaTs/UlE/50NcJ+IPC0iS+22in23R1yinFI+jDFG\nRCIZtiYiY4HbgU8aY7pFJH0siu9tjEkCC0VkPPBb4Mgqd6msiMhbgB3GmKdFZEm1+1NBFhtjtojI\nZOB+EXnRfbDc3+1a1CDyFgSMGNtFZCqA/f8Ouz0yvwcRqccSDrcYY+6wmyP/3gDGmD3AQ1jmlfF2\nPTPwvlf6ne3j44CuCnf1QDkFuFBENmKtE3MG8L9E+50xxmyx/9+BNRE4kQp+t2tRQPwdmGtHPzQA\n7wZ+V+U+lZPfAZfa25di2eid9kvsyIeTgL0utXXUIJaqcCOw2hhzretQZN9bRDpszQERacbyuazG\nEhTvsE/zv7Pzu3gH8KCxjdSjBWPMFcaYGcaY2Vh/sw8aY/6JCL+ziLSISKuzDZwFrKSS3+1qO2Gq\n5Pg5D6tq7Hrgc9XuzzC+163AVmAIy/74ASy76wPAWuDPwET7XMGK5loPPA8sqnb/S3znxVh22hXA\ncvvnvCi/N3AM8Kz9ziuBq+z2Q4CngHXAr4FGu73J3l9nHz+k2u9wgO+/BLg76u9sv9tz9s8LzlhV\nye+2ZlIriqIogdSiiUlRFEUpABUQiqIoSiAqIBRFUZRAVEAoiqIogaiAUBRFUQJRAaHUJCKStCtk\nOj85q/qKyIdF5JJheO5GEWkv4bqzReRqu5LnHw+0H4pSCFpqQ6lV9htjFhZ6sjHm/8rZmQI4FSsp\n7FTgsSr3RakRVINQFBf2DP/rdg3+p0TkMLv9SyLyGXv7MrHWn1ghIr+02yaKyJ122xMicozdPklE\n7hNr3YYfYSUzOc96r/2M5SLyQ7sUvb8/77KL8l2GVazuBuD9IhLl7H9lhKACQqlVmn0mpne5ju01\nxhwNXIc1KPu5HDjOGHMM8GG77WrgWbvtSuBmu/2LwGPGmPlYtXRmAYjIUcC7gFNsTSYJ/JP/QcaY\n27Aq1K60+/S8/ewLD+TlFaUQ1MSk1Cq5TEy3uv7/dsDxFcAtInIncKfdthj4BwBjzIO25tAGnAa8\n3W7/g4jsts8/Ezge+LtdebaZTNE1P4cDG+ztFmPMvgLeT1EOGBUQipKNCdl2OB9r4L8A+JyIHF3C\nMwT4qTHmipwnWctMtgN1IrIKmGqbnD5ujPlLCc9VlIJRE5OiZPMu1/+Puw+ISAyYaYx5CPgsVhnp\nscBfsE1E9noFO40x3cCjwHvs9nOBCfatHgDeYdf5d3wYB/s7YqxlJv8AXAR8Hatg20IVDkolUA1C\nqVWa7Zm4w73GGCfUdYKIrMBa+/li33Vx4OciMg5LC/iuMWaPiHwJuMm+ro9MOeargVtF5AXgb8Am\nAGPMKhH5PNZqYTGsCrwfA14J6OvrsJzUHwWuDTiuKGVBq7kqigt7QZpFxpid1e6LolQbNTEpiqIo\ngagGoSiKogSiGoSiKIoSiAoIRVEUJRAVEIqiKEogKiAURVGUQFRAKIqiKIGogFAURVEC+f8f866N\nzGa41QAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o2VBDwpLpfjd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AdGMwOMUzp_z",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "61c44e0f-7811-401c-ac4f-2198bd7b69e6"
      },
      "source": [
        "agent.qnetwork_local.noisy_layer_sigma_snr()"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[23.257034301757812, 6.289618492126465, 6.674216270446777, 8.123523712158203]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NFOs2c0B0-EN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}