PPLM_paraphrase.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# %load_ext autoreload\n",
    "# %autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 683
    },
    "colab_type": "code",
    "id": "dQ9z0QkTWt1V",
    "outputId": "ed2430e6-bebc-4466-c44c-f7810997c7fb",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DfNwx_F_YZXE",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from run_pplm_paraphrase import run_pplm_example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import GPT2Tokenizer\n",
    "from transformers.modeling_gpt2 import GPT2LMHeadModel\n",
    "\n",
    "pretrained_model=\"gpt2-medium\"\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "model = GPT2LMHeadModel.from_pretrained(\n",
    "    pretrained_model,\n",
    "    output_hidden_states=True\n",
    ")\n",
    "model.to(device)\n",
    "model.eval()\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = False\n",
    "\n",
    "tokenizer = GPT2Tokenizer.from_pretrained(pretrained_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "B5Yl_u6BYnS5"
   },
   "source": [
    "# Let's generate some text!\n",
    "When you specify a number of samples, PPLM will first generate a sample without any modification for reference (called \"unperturbed\"), and then the number of samples specified (called \"perturbed\"). If you want to generate different samples given the same parameters, change the `seed` value to a number other than 0 (the default value)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "phrase = \"Endemic types or species are especially likely to develop on islands.\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ниже примеры генерации **PPLM** фраз на основе исходной. В отдельных случаях подбираются подходящие синонимы, но в целом это скорее генерация на заданную тему, чем перефразирование."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.04 | 8 | 0.001 | 1 |\n",
      "Endemic types of bacteria are particularly likely to develop resistance to antibiotics\n",
      "<|endoftext|>Endemic types of bacteria are particularly likely to develop resistance to antibiotics\n",
      "\n",
      "0.04 | 12 | 0.001 | 1 |\n",
      "Endemic types or species are the most common and most diverse.\n",
      "<|endoftext|>Endemic types or species are the most common and most diverse.\n",
      "\n",
      "0.04 | 12 | 0.001 | 3 |\n",
      "Endemic forms of cancer are especially likely to develop among women.\n",
      "<|endoftext|>Endemic forms of cancer are especially likely to develop among women.\n",
      "\n",
      "0.04 | 8 | 1e-05 | 3 |\n",
      "Endemic diseases, including diabetes, have to develop on islands.\n",
      "<|endoftext|>Endemic diseases, including diabetes, have to develop on islands.\n",
      "\n",
      "0.04 | 12 | 1e-05 | 3 |\n",
      "Endemic forms of species are especially likely to develop on islands.\n",
      "<|endoftext|>Endemic forms of species are especially likely to develop on islands.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "stepsize = 0.04\n",
    "\n",
    "for num_iterations, kl_scale, top_k in [\n",
    "    (8, 0.001, 1), \n",
    "    (12, 0.001, 1), \n",
    "    (12, 0.001, 3),\n",
    "    (8, 1e-05, 3),\n",
    "    (12, 1e-05, 3)\n",
    "]:\n",
    "    print(stepsize, '|', num_iterations, '|', kl_scale, '|', top_k, '|')\n",
    "    _, _ = run_pplm_example(\n",
    "        model, tokenizer, device,\n",
    "        cond_text=phrase,\n",
    "        stepsize=stepsize,\n",
    "        num_iterations=num_iterations,\n",
    "        kl_scale=kl_scale,\n",
    "        top_k=top_k\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1 | 4 | 0.01 | 1 |\n",
      "Endemic diseases are the leading causes of death in the world.\n",
      "<|endoftext|>Endemic diseases are the leading causes of death in the world.\n",
      "\n",
      "0.1 | 12 | 0.01 | 1 |\n",
      "Endemic types or species are the most common types on islands.\n",
      "<|endoftext|>Endemic types or species are the most common types on islands.\n",
      "\n",
      "0.1 | 4 | 0.001 | 1 |\n",
      "Endemic types of bacteria are particularly likely to develop resistance to antibiotics\n",
      "<|endoftext|>Endemic types of bacteria are particularly likely to develop resistance to antibiotics\n",
      "\n",
      "0.1 | 6 | 0.0001 | 1 |\n",
      "Endemic types or species are particularly likely to develop on islands.\n",
      "<|endoftext|>Endemic types or species are particularly likely to develop on islands.\n",
      "\n",
      "0.1 | 8 | 1e-05 | 1 |\n",
      "Endemic types or species are especially likely to develop cancer..\n",
      "<|endoftext|>Endemic types or species are especially likely to develop cancer..\n",
      "\n"
     ]
    }
   ],
   "source": [
    "stepsize = 0.1\n",
    "top_k = 1\n",
    "\n",
    "for num_iterations, kl_scale in [\n",
    "    (4, 1e-2),\n",
    "    (12, 1e-2),\n",
    "    (4, 1e-3), \n",
    "    (6, 1e-4),\n",
    "    (8, 1e-5)\n",
    "]:\n",
    "    print(stepsize, '|', num_iterations, '|', kl_scale, '|', top_k, '|')\n",
    "    _, _ = run_pplm_example(\n",
    "        model, tokenizer, device,\n",
    "        cond_text=phrase,\n",
    "        stepsize=stepsize,\n",
    "        num_iterations=num_iterations,\n",
    "        kl_scale=kl_scale,\n",
    "        top_k=top_k\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Четвёртый пример (0.1, 6, 1e-4, 1):  \n",
    "    `Endemic types or species are particularly likely to develop on islands.` - единственный случай с сохранением смысла."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.2 | 4 | 0.001 | 1 |\n",
      "Endemic types or species are found in the world on all continents\n",
      "<|endoftext|>Endemic types or species are found in the world on all continents\n",
      "\n",
      "0.2 | 4 | 0.001 | 2 |\n",
      "Endemic forms of species are the likely to develop in the future\n",
      "<|endoftext|>Endemic forms of species are the likely to develop in the future\n",
      "\n"
     ]
    }
   ],
   "source": [
    "stepsize = 0.2\n",
    "\n",
    "for num_iterations, kl_scale, top_k in [\n",
    "    (4, 1e-3, 1),\n",
    "    (4, 1e-3, 2)\n",
    "]:\n",
    "    print(stepsize, '|', num_iterations, '|', kl_scale, '|', top_k, '|')\n",
    "    _, _ = run_pplm_example(\n",
    "        model, tokenizer, device,\n",
    "        cond_text=phrase,\n",
    "        stepsize=stepsize,\n",
    "        num_iterations=num_iterations,\n",
    "        kl_scale=kl_scale,\n",
    "        top_k=top_k\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "При некоторых ограничениях на параметры генерируется точная копия фразы."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.04 | 24 | 0.0001 | 1 |\n",
      "Endemic types or species are especially likely to develop on islands.\n",
      "<|endoftext|>Endemic types or species are especially likely to develop on islands.\n",
      "\n",
      "0.04 | 12 | 1e-05 | 1 |\n",
      "Endemic types or species are especially likely to develop on islands.\n",
      "<|endoftext|>Endemic types or species are especially likely to develop on islands.\n",
      "\n",
      "0.1 | 12 | 1e-05 | 1 |\n",
      "Endemic types or species are especially likely to develop on islands.\n",
      "<|endoftext|>Endemic types or species are especially likely to develop on islands.\n",
      "\n",
      "0.1 | 8 | 1e-06 | 1 |\n",
      "Endemic types of species are especially likely to develop on islands.\n",
      "<|endoftext|>Endemic types of species are especially likely to develop on islands.\n",
      "\n",
      "0.1 | 24 | 0.0001 | 1 |\n",
      "Endemic types or species are especially likely to develop on islands.\n",
      "<|endoftext|>Endemic types or species are especially likely to develop on islands.\n",
      "\n",
      "0.2 | 8 | 0.0001 | 1 |\n",
      "Endemic types or species are especially likely to develop on islands.\n",
      "<|endoftext|>Endemic types or species are especially likely to develop on islands.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "top_k = 1\n",
    "\n",
    "for stepsize, num_iterations, kl_scale in [\n",
    "    (0.04, 24, 1e-4),\n",
    "    (0.04, 12, 1e-5),\n",
    "    (0.1, 12, 1e-5),\n",
    "    (0.1, 6, 1e-6),\n",
    "    (0.1, 24, 1e-4),\n",
    "    (0.2, 6, 1e-4),\n",
    "]:\n",
    "    print(stepsize, '|', num_iterations, '|', kl_scale, '|', top_k, '|')\n",
    "    _, _ = run_pplm_example(\n",
    "        model, tokenizer, device,\n",
    "        cond_text=phrase,\n",
    "        stepsize=stepsize,\n",
    "        num_iterations=num_iterations,\n",
    "        kl_scale=kl_scale,\n",
    "        top_k=top_k\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Предложения из комментариев к коду."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "phrase = \"The discriminator model is more complex. \\\n",
    "It takes both real image samples and random noise seeds as input.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.2 | 18 | 0.001 | 1 |\n",
      "The first thing you notice more complex. It takes both real-time and random noise to as input.\n",
      "<|endoftext|>The first thing you notice more complex. It takes both real-time and random noise to as input.\n",
      "\n",
      "0.2 | 7 | 0.0001 | 1 |\n",
      "The discriminator model is more complex. It takes both real-world and random noise into as input.\n",
      "<|endoftext|>The discriminator model is more complex. It takes both real-world and random noise into as input.\n",
      "\n",
      "0.2 | 8 | 0.0001 | 1 |\n",
      "The discriminator model is more complex. It takes both real world samples and random noise to as input.\n",
      "<|endoftext|>The discriminator model is more complex. It takes both real world samples and random noise to as input.\n",
      "\n",
      "0.1 | 12 | 1e-05 | 1 |\n",
      "The discriminator model is more complex. It takes both real image samples and random noise seeds as input.\n",
      "<|endoftext|>The discriminator model is more complex. It takes both real image samples and random noise seeds as input.\n",
      "\n",
      "0.2 | 12 | 1e-05 | 1 |\n",
      "The discriminator model is more complex. It takes both real image samples and random noise seeds as input.\n",
      "<|endoftext|>The discriminator model is more complex. It takes both real image samples and random noise seeds as input.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "top_k = 1\n",
    "\n",
    "for stepsize, num_iterations, kl_scale in [\n",
    "    (0.2, 18, 1e-3),\n",
    "    (0.2, 7, 1e-4),\n",
    "    (0.2, 8, 1e-4),\n",
    "    (0.1, 12, 1e-5),\n",
    "    (0.2, 12, 1e-5)\n",
    "]:\n",
    "    print(stepsize, '|', num_iterations, '|', kl_scale, '|', top_k, '|')\n",
    "    _, _ = run_pplm_example(\n",
    "        model, tokenizer, device,\n",
    "        cond_text=phrase,\n",
    "        stepsize=stepsize,\n",
    "        num_iterations=num_iterations,\n",
    "        kl_scale=kl_scale,\n",
    "        top_k=top_k\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "phrase = \"The GPT2 Model transformer with a language modeling head on top \\\n",
    "(linear layer with weights tied to the input embeddings).\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1 | 22 | 1e-05 | 1 |\n",
      "The GPT2 Model is with a language modeling head on top (linear layer with layers tied to the input embeddings).\n",
      "<|endoftext|>The GPT2 Model is with a language modeling head on top (linear layer with layers tied to the input embeddings).\n",
      "\n",
      "0.1 | 24 | 1e-05 | 1 |\n",
      "The GPT2 Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings).\n",
      "<|endoftext|>The GPT2 Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings).\n",
      "\n"
     ]
    }
   ],
   "source": [
    "top_k = 1\n",
    "\n",
    "for stepsize, num_iterations, kl_scale in [\n",
    "    (0.1, 22, 1e-5),\n",
    "    (0.1, 24, 1e-5) \n",
    "]:\n",
    "    print(stepsize, '|', num_iterations, '|', kl_scale, '|', top_k, '|')\n",
    "    _, _ = run_pplm_example(\n",
    "        model, tokenizer, device,\n",
    "        cond_text=phrase,\n",
    "        stepsize=stepsize,\n",
    "        num_iterations=num_iterations,\n",
    "        kl_scale=kl_scale,\n",
    "        top_k=top_k\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "И посмотрим на обычную речь."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "phrase = \"I couldn't bear to watch it. And I thought the UA loss was embarrassing\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.04 | 16 | 0.001 | 1 |\n",
      "I couldn't bear to watch it. And I thought the movie was awful.\n",
      "<|endoftext|>I couldn't bear to watch it. And I thought the movie was awful.\n",
      "\n",
      "0.04 | 16 | 0.0001 | 1 |\n",
      "I couldn't bear to watch it. And I thought the movie was terrible.\n",
      "<|endoftext|>I couldn't bear to watch it. And I thought the movie was terrible.\n",
      "\n",
      "0.1 | 8 | 0.0001 | 1 |\n",
      "I couldn't bear to watch it. And I thought the scene was just embarrassing\n",
      "<|endoftext|>I couldn't bear to watch it. And I thought the scene was just embarrassing\n",
      "\n",
      "0.1 | 12 | 0.001 | 1 |\n",
      "I couldn't bear to watch it. And I thought the movie was pretty awful\n",
      "<|endoftext|>I couldn't bear to watch it. And I thought the movie was pretty awful\n",
      "\n",
      "0.2 | 8 | 1e-06 | 1 |\n",
      "I couldn't bear to watch it. And I thought the Trump loss was embarrassing\n",
      "<|endoftext|>I couldn't bear to watch it. And I thought the Trump loss was embarrassing\n",
      "\n"
     ]
    }
   ],
   "source": [
    "top_k = 1\n",
    "\n",
    "for stepsize, num_iterations, kl_scale in [\n",
    "    (0.04, 16, 1e-3),\n",
    "    (0.04, 16, 1e-4),\n",
    "    (0.1, 8, 1e-4),\n",
    "    (0.1, 12, 1e-3),\n",
    "    (0.2, 8, 1e-6)\n",
    "]:\n",
    "    print(stepsize, '|', num_iterations, '|', kl_scale, '|', top_k, '|')\n",
    "    _, _ = run_pplm_example(\n",
    "        model, tokenizer, device,\n",
    "        cond_text=phrase,\n",
    "        stepsize=stepsize,\n",
    "        num_iterations=num_iterations,\n",
    "        kl_scale=kl_scale,\n",
    "        top_k=top_k\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1 | 20 | 0.0001 | 1 |\n",
      "I couldn't bear to watch it. And I thought the whole loss was embarrassing\n",
      "<|endoftext|>I couldn't bear to watch it. And I thought the whole loss was embarrassing\n",
      "\n",
      "0.1 | 12 | 1e-06 | 1 |\n",
      "I couldn't bear to watch it. And I thought the same loss was embarrassing\n",
      "<|endoftext|>I couldn't bear to watch it. And I thought the same loss was embarrassing\n",
      "\n",
      "0.1 | 24 | 0.0001 | 1 |\n",
      "I couldn't bear to watch it. And I thought the UA loss was embarrassing\n",
      "<|endoftext|>I couldn't bear to watch it. And I thought the UA loss was embarrassing\n",
      "\n",
      "0.2 | 18 | 1e-06 | 1 |\n",
      "I couldn't bear to watch it. And I thought the UA loss was embarrassing\n",
      "<|endoftext|>I couldn't bear to watch it. And I thought the UA loss was embarrassing\n",
      "\n"
     ]
    }
   ],
   "source": [
    "top_k = 1\n",
    "\n",
    "for stepsize, num_iterations, kl_scale in [\n",
    "    (0.1, 20, 1e-4),\n",
    "    (0.1, 12, 1e-6),\n",
    "    (0.1, 24, 1e-4),\n",
    "    (0.2, 18, 1e-6),\n",
    "]:\n",
    "    print(stepsize, '|', num_iterations, '|', kl_scale, '|', top_k, '|')\n",
    "    _, _ = run_pplm_example(\n",
    "        model, tokenizer, device,\n",
    "        cond_text=phrase,\n",
    "        stepsize=stepsize,\n",
    "        num_iterations=num_iterations,\n",
    "        kl_scale=kl_scale,\n",
    "        top_k=top_k\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Табличка из wiki\n",
    "\n",
    "- Original (biology): `Endemic types or species are especially likely to develop on islands.`  \n",
    "\n",
    "| stepsize      | num_iterations | kl_scale  | top_k | Generated phrase |\n",
    "| ------------- |:-------------:| -----:|-----:|-----:|\n",
    "0.04 | 8 | 1e-03 | 1 | Endemic types of bacteria are particularly likely to develop resistance to antibiotics\n",
    "0.04 | 8 | 1e-05 | 3 | Endemic diseases, including diabetes, have to develop on islands.\n",
    "0.04 | 12 | 1e-05 | 3 | Endemic forms of species are especially likely to develop on islands.\n",
    "0.1 | 4 | 0.01 | 1 | Endemic diseases are the leading causes of death in the world.\n",
    "0.1 | 12 | 0.01 | 1 | Endemic types or species are the most common types on islands.\n",
    "0.1 | 6 | 1e-04 | 1 | Endemic types or species are particularly likely to develop on islands.\n",
    "0.2 | 8 | 1e-04 | 1 | Endemic types or species are especially likely to develop on islands.\n",
    "\n",
    "- Originals (technical): `\"The discriminator model is more complex. It takes both real image samples and random noise seeds as input.\"`  \n",
    "and `\"The GPT2 Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings).\"`  \n",
    "\n",
    "| stepsize      | num_iterations | kl_scale  | top_k | Generated phrase |\n",
    "| ------------- |:-------------:| -----:|-----:|-----:|\n",
    "0.2 | 7 | 1e-04 | 1 | The discriminator model is more complex. It takes both real-world and random noise into as input.\n",
    "0.1 | 12 | 1e-05 | 1 | The discriminator model is more complex. It takes both real image samples and random noise seeds as input.\n",
    "0.1 | 22 | 1e-05 | 1 | The GPT2 Model is with a language modeling head on top (linear layer with layers tied to the input embeddings).\n",
    "0.1 | 24 | 1e-05 | 1 | The GPT2 Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings).\n",
    "\n",
    "- Original (informal speech): `\"I couldn't bear to watch it. And I thought the UA loss was embarrassing\"`\n",
    "\n",
    "| stepsize      | num_iterations | kl_scale  | top_k | Generated phrase |\n",
    "| ------------- |:-------------:| -----:|-----:|-----:|\n",
    "0.04 | 16 | 1e-03 | 1 | I couldn't bear to watch it. And I thought the movie was awful.\n",
    "0.04 | 16 | 1e-04 | 1 | I couldn't bear to watch it. And I thought the movie was terrible.\n",
    "0.1 | 8 | 1e-04 | 1 | I couldn't bear to watch it. And I thought the scene was just embarrassing\n",
    "0.2 | 8 | 1e-06 | 1 | I couldn't bear to watch it. And I thought the Trump loss was embarrassing\n",
    "=== | == | ===== | = | =========================================\n",
    "0.1 | 20 | 1e-04 | 1 | I couldn't bear to watch it. And I thought the whole loss was embarrassing\n",
    "0.1 | 12 | 1e-06 | 1 | I couldn't bear to watch it. And I thought the same loss was embarrassing\n",
    "0.1 | 24 | 1e-04 | 1 | I couldn't bear to watch it. And I thought the UA loss was embarrassing\n",
    "0.2 | 18 | 1e-06 | 1 | I couldn't bear to watch it. And I thought the UA loss was embarrassing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Copy of PPLM.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}