unsupervised_extractive_summarization_with_bert.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "unsupervised_extractive_summarization_with_bert.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/marucha80t/use-cases_of_bert/blob/master/unsupervised_extractive_summarization_with_bert.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ICvreRPWZHlM",
        "colab_type": "text"
      },
      "source": [
        "# Unsupervised Extractive Summarization with BERT\n",
        "This notebook demonstrates EmbedRank, which is an unsupervised keyphrase extraction model [1]. Sentence embeddings obtained from Hugging Face's BERT are used to calculate each sentence's importance.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zzp224RYZedC",
        "colab_type": "text"
      },
      "source": [
        "## Settings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hHzpRs-jZOof",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "cellView": "form",
        "outputId": "712e9fe1-f4c5-41f3-87da-e3d72e59bddb"
      },
      "source": [
        "#@title Setup Environment\n",
        "!pip install --quiet googletrans==2.4.0\n",
        "!pip install --quiet japanize-matplotlib==1.0.4\n",
        "!pip install --quiet pytorch_transformers\n",
        "!pip install --quiet mecab-python3\n",
        "!pip install --quiet https://github.com/megagonlabs/ginza/releases/download/v1.0.2/ja_ginza_nopn-1.0.2.tgz\n",
        "!pip install --quiet https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.1.0/en_core_web_sm-2.1.0.tar.gz\n",
        "!ln -s /usr/local/lib/python3.6/dist-packages/ja_ginza_nopn /usr/local/lib/python3.6/dist-packages/spacy/data/ja_ginza_nopn"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 122.4MB 312kB/s \n",
            "\u001b[?25h  Building wheel for ja-ginza-nopn (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[K     |████████████████████████████████| 11.1MB 1.7MB/s \n",
            "\u001b[?25h  Building wheel for en-core-web-sm (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "ln: failed to create symbolic link '/usr/local/lib/python3.6/dist-packages/spacy/data/ja_ginza_nopn/ja_ginza_nopn': File exists\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ukbf524JZjRg",
        "colab_type": "code",
        "colab": {},
        "cellView": "form"
      },
      "source": [
        "#@title Setup common imports and functions\n",
        "from googletrans import Translator\n",
        "import torch\n",
        "from pytorch_transformers import (\n",
        "    BertTokenizer,\n",
        "    BertModel,\n",
        ")\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import japanize_matplotlib\n",
        "import seaborn as sns\n",
        "import spacy\n",
        "from IPython.display import HTML\n",
        "from sklearn import manifold\n",
        "from sklearn.metrics.pairwise import cosine_distances\n",
        "%matplotlib inline\n",
        "\n",
        "\n",
        "def ncossim(embs_1, embs_2, axis=0):\n",
        "    sims = np.inner(embs_1, embs_2)\n",
        "    std = np.std(sims, axis=axis)\n",
        "    ex = np.mean((sims-np.min(sims, axis=axis))/np.max(sims, axis=axis), axis=axis)\n",
        "    return 0.5 + (sims-ex)/std\n",
        "\n",
        "\n",
        "def mmr(doc_emb, cand_embs, key_embs):\n",
        "    param = 0.5\n",
        "    scores = param * ncossim(cand_embs, doc_emb, axis=0)\n",
        "    if key_embs is not None:\n",
        "        scores -= (1-param) * np.max(ncossim(cand_embs, key_embs), axis=1).reshape(scores.shape[0], -1)\n",
        "    return scores\n",
        "\n",
        "\n",
        "def embedrank(doc_emb, sent_embs, n_keys):\n",
        "    assert 0 < n_keys, 'Please `key_size` value set more than 0'\n",
        "    assert n_keys < len(sent_embs), 'Please `key_size` value set lower than `#sentences`'\n",
        "    sims = np.inner(doc_emb, sent_embs).reshape(-1)\n",
        "    return np.argsort(-sims)[:n_keys]\n",
        "\n",
        "\n",
        "def embedrankpp(doc_emb, sent_embs, n_keys):\n",
        "    assert 0 < n_keys, 'Please `key_size` value set more than 0'\n",
        "    assert n_keys < len(sent_embs), 'Please `key_size` value set lower than `#sentences`'\n",
        "    cand_idx = list(range(len(sent_embs)))\n",
        "    key_idx = []\n",
        "    while len(key_idx) < n_keys:\n",
        "        cand_embs = sent_embs[cand_idx]\n",
        "        key_embs = sent_embs[key_idx] if len(key_idx) > 0 else None\n",
        "        scores = mmr(doc_emb, cand_embs, key_embs)\n",
        "        key_idx.append(cand_idx[np.argmax(scores)])\n",
        "        cand_idx.pop(np.argmax(scores))\n",
        "    return key_idx"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1lWmmj1raL4l",
        "colab_type": "text"
      },
      "source": [
        "## Extractive Summarization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GVZPWVlsdnFn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Document from Wikipedia\n",
        "doc = \"\"\"\n",
        "自動要約（じどうようやく）は、コンピュータプログラムを用いて、文書からその要約を作成する処理である。\n",
        "作成される要約は、要約の対象となる文書の最も重要な要素のみを残しているべきであり、いわゆる情報のオーバーロードに伴い自動要約に対する関心も増している。\n",
        "首尾一貫した要約を作成するためには要約の長さや書き方のスタイル、文法などといった点が考慮されなければならない。\n",
        "自動要約の応用先の1つはGoogleなどの検索エンジンであるが、もちろん独立した1つの要約プログラムといったものもありうる。\n",
        "自動要約は、要約の目的や要約の対象とする文書の数、要約の方法などによっていくつかの種類に分類することができる。\n",
        "抽出的要約は、要約の対象となる文書に含まれる単語や句、文といった単位をうまく抽出し、それらを組み合わせることで要約を作成する。\n",
        "一方、生成的要約は、文書を一度何らかの中間表現（あるいは意味表現）に変換し、この中間表現を元に自然言語生成の技術を用いて要約を作成する。\n",
        "そのため、生成的要約によって作成された要約には元の文書に含まれていない表現が含まれることもありうる。\n",
        "生成的要約には、文書を中間表現に正確に変換すること（すなわち、精度の高い自然言語理解を実現すること）、そこから要約を生成するための自然言語生成器が必要になるといった問題が存在するため、もっぱら研究の焦点は抽出的要約にあてられている。\n",
        "\"\"\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8nSSOvQZuQZj",
        "colab_type": "code",
        "cellView": "form",
        "outputId": "e84c58f0-7938-4ae2-edd8-7da05351129a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "#@title Language detection and sentence segmentation\n",
        "translator = Translator()\n",
        "detected_lang = translator.detect(doc)\n",
        "\n",
        "assert detected_lang.lang in ['ja', 'en'], 'Please, input Japanese text or English text'\n",
        "if detected_lang.lang == 'ja':\n",
        "    sentence_splitter = spacy.load('ja_ginza_nopn')\n",
        "elif detected_lang.lang == 'en':\n",
        "    sentence_splitter = spacy.load('en_core_web_sm')\n",
        "\n",
        "sents = [str(s) for s in sentence_splitter(doc.replace('\\n', '')).sents]\n",
        "print(f'Language: {detected_lang.lang}')\n",
        "print(f'#sentences: {len(sents)}')"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Language: ja\n",
            "#sentences: 9\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "no3oU6hfaNIG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "cellView": "form",
        "outputId": "76e6c541-56f8-49c2-b2a7-fbcade63d0a8"
      },
      "source": [
        "#@title Build a model\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')\n",
        "model = BertModel.from_pretrained('bert-base-multilingual-cased')\n",
        "rank_fn = embedrankpp"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The pre-trained model you are loading is a cased model but you have not set `do_lower_case` to False. We are setting `do_lower_case=False` for you but you may want to check this behavior.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kgRNVcBFcY9O",
        "colab_type": "code",
        "colab": {},
        "cellView": "form"
      },
      "source": [
        "#@title Model run\n",
        "# Convert tokens into ids\n",
        "encoded_doc = torch.tensor(tokenizer.encode(doc)).unsqueeze(0)\n",
        "encoded_sents = [tokenizer.encode(s) for s in sents]\n",
        "pad_idx = tokenizer.convert_tokens_to_ids(tokenizer.pad_token)\n",
        "maxlen = max([len(s) for s in encoded_sents])\n",
        "encoded_sents = torch.tensor([s + [pad_idx for _ in range(maxlen-len(s))] for s in encoded_sents])\n",
        "\n",
        "if torch.cuda.is_available:\n",
        "    model.to('cuda')\n",
        "    encoded_doc = encoded_doc.to('cuda')\n",
        "    encoded_sents = encoded_sents.to('cuda')\n",
        "    \n",
        "# Encode\n",
        "doc_emb = torch.mean(model(encoded_doc)[0], dim=1).to('cpu').detach().numpy()\n",
        "sent_embs = torch.mean(model(encoded_sents)[0], dim=1).to('cpu').detach().numpy()\n",
        "\n",
        "# Ranking\n",
        "key_size = 3 #@param {type:\"integer\"}\n",
        "keys = rank_fn(doc_emb, sent_embs, key_size)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b5HKVZd8s4ct",
        "colab_type": "code",
        "cellView": "both",
        "outputId": "a8ef6658-a0c7-491f-cf3d-a8535a0b2cb0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        }
      },
      "source": [
        "#@title Display\n",
        "display_sents = []\n",
        "for i, s in enumerate(sents):\n",
        "    line = '<font color=\"#CD5C5C\"><strong>' + s + '</strong></font>' if i in keys else s\n",
        "    display_sents.append(line)\n",
        "HTML(''.join(display_sents))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<font color=\"#CD5C5C\"><strong>自動要約（じどうようやく）は、コンピュータプログラムを用いて、文書からその要約を作成する処理である。</strong></font>作成される要約は、要約の対象となる文書の最も重要な要素のみを残しているべきであり、いわゆる情報のオーバーロードに伴い自動要約に対する関心も増している。首尾一貫した要約を作成するためには要約の長さや書き方のスタイル、文法などといった点が考慮されなければならない。自動要約の応用先の1つはGoogleなどの検索エンジンであるが、もちろん独立した1つの要約プログラムといったものもありうる。<font color=\"#CD5C5C\"><strong>自動要約は、要約の目的や要約の対象とする文書の数、要約の方法などによっていくつかの種類に分類することができる。</strong></font>抽出的要約は、要約の対象となる文書に含まれる単語や句、文といった単位をうまく抽出し、それらを組み合わせることで要約を作成する。一方、生成的要約は、文書を一度何らかの中間表現（あるいは意味表現）に変換し、この中間表現を元に自然言語生成の技術を用いて要約を作成する。そのため、生成的要約によって作成された要約には元の文書に含まれていない表現が含まれることもありうる。<font color=\"#CD5C5C\"><strong>生成的要約には、文書を中間表現に正確に変換すること（すなわち、精度の高い自然言語理解を実現すること）、そこから要約を生成するための自然言語生成器が必要になるといった問題が存在するため、もっぱら研究の焦点は抽出的要約にあてられている。</strong></font>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "47H6fAx6qdRt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 462
        },
        "cellView": "form",
        "outputId": "43b8e35b-cfab-466d-e41b-543da15296fb"
      },
      "source": [
        "#@title Visualize sentence embeddings\n",
        "# print sentences\n",
        "print('id' + '\\t' + 'sentence')\n",
        "for i, sent in enumerate(sents, 1):\n",
        "    if len(sent) < 50:\n",
        "        print(str(i) + '\\t' + sent)\n",
        "    else:\n",
        "        print(str(i) + '\\t' + sent[:50] + '...')\n",
        "print('')\n",
        "\n",
        "# MDS\n",
        "mds = manifold.MDS(n_components=2, dissimilarity=\"precomputed\")\n",
        "embs = np.concatenate((doc_emb, sent_embs), 0)\n",
        "dist_matrix = cosine_distances(embs, embs)\n",
        "pns = mds.fit_transform(dist_matrix)\n",
        "fixed_pns = pns - pns[0]\n",
        "# plot\n",
        "keys_idx = [idx + 1 for idx in keys]\n",
        "other_idx = [idx for idx in range(1, len(sents)+1) if idx not in keys_idx]\n",
        "plt.scatter(fixed_pns[0,0], fixed_pns[0,1], color='green',  marker='*', s=150, label='document')\n",
        "plt.scatter(fixed_pns[keys_idx,0], fixed_pns[keys_idx, 1], color='blue', label='key sentences')\n",
        "plt.scatter(fixed_pns[other_idx,0], fixed_pns[other_idx, 1], color='white', edgecolors='black', label='other sentences')\n",
        "\n",
        "for i,(x,y) in enumerate(fixed_pns[1:], 1):\n",
        "    plt.annotate(str(i), (x+0.002 ,y+0.002))\n",
        "    \n",
        "plt.legend()\n",
        "plt.show()\n"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "id\tsentence\n",
            "1\t自動要約（じどうようやく）は、コンピュータプログラムを用いて、文書からその要約を作成する処理である。...\n",
            "2\t作成される要約は、要約の対象となる文書の最も重要な要素のみを残しているべきであり、いわゆる情報のオー...\n",
            "3\t首尾一貫した要約を作成するためには要約の長さや書き方のスタイル、文法などといった点が考慮されなければ...\n",
            "4\t自動要約の応用先の1つはGoogleなどの検索エンジンであるが、もちろん独立した1つの要約プログラム...\n",
            "5\t自動要約は、要約の目的や要約の対象とする文書の数、要約の方法などによっていくつかの種類に分類すること...\n",
            "6\t抽出的要約は、要約の対象となる文書に含まれる単語や句、文といった単位をうまく抽出し、それらを組み合わ...\n",
            "7\t一方、生成的要約は、文書を一度何らかの中間表現（あるいは意味表現）に変換し、この中間表現を元に自然言...\n",
            "8\tそのため、生成的要約によって作成された要約には元の文書に含まれていない表現が含まれることもありうる。...\n",
            "9\t生成的要約には、文書を中間表現に正確に変換すること（すなわち、精度の高い自然言語理解を実現すること）...\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD7CAYAAACG50QgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl4VeW59/HvTQiTUKkhCAUJHBUH\nkFINKAYIOBxxQBwqlVoEjKa1l4gKeuTN5QSirRWO74tHARmliFppiVUErSQMgkCgqUOFVqtgKngC\nZRBCBsj9/pGdNITA3oEkO8n6fa5rXzv7WdO9V5JfVp691rPM3RERkWBoFO0CRESk9ij0RUQCRKEv\nIhIgCn0RkQBR6IuIBIhCX0QkQBT6IiIBotAXEQkQhb6ISIA0jnYBFbVp08Y7d+4c7TJEROqVjRs3\n7nT3+HDz1bnQ79y5M1lZWdEuQ0SkXjGzrZHMp+4dEZEAUeiLiASIQl9EJEDqXJ9+ZYqKisjJySE/\nPz/apUgVNGvWjI4dOxIbGxvtUkTqhaeeeorFixcTGxvLD37wA2bPnk2rVq2qdRv1IvRzcnJo1aoV\nnTt3xsyiXY5EwN3ZtWsXOTk5dOnSJdrliNR5H3/8Menp6axdu5aYmBjuv/9+pk2bxoMPPlit26kX\n3Tv5+fnExcVVOfALDxfWUEUSjpkRFxen/85EItSmTRuaNm3KoUOHADh8+DA9e/as9u3UiyN9oMqB\nn3sgl3OeP4ct92wh/pSwp65KDdB/ZSKRa9++Pffccw+//OUvOeuss/j+97/PFVdcUe3bqRdH+idi\n8ebF7M7fTfqW9GiXIiISVkZGBitXrmTWrFmMHz+ebt268dhjj1X7dhps6M/JnnPEc3V6/PHHmTZt\nWrWvt7r84x//4Jtvvol2GSJyHAsXLqR79+7ExMTQvXt35s+fT0FBQdn0wsJC/v73v1f7dutN905V\n7D64m43bNwKQ9U0We/L30LpZ6yhXVXsmTJjAyJEj+cEPfhDtUkSkEgsXLiQtLY1Zs2bRt29fVq9e\nzahRo+jUqRO9e/cmNjaW5s2bM3PmzGrfdoMI/dQ/pvLGX9/AcQAOFR8itlEshYcLiW0Uyxn/fQaN\nG5W8VcO45fxbmD54epW2MXbsWFavXl12CmK7du3Iyspi7NixmBmtWrXipZdeol27dqxbt44HHniA\n4uJizjjjDObOncvrr7/O5s2b+dWvfgVAx44dycnJITMzkylTptC4cWO2bNnChAkTmDFjBtu3b2fq\n1KkkJyezY8cOUlJS2L9/P9/73veYO3cucXFxnHvuufz0pz8lMzOTvXv3kp6eztdff83SpUvJzs7m\n1ltv5eGHH67enS0iJ23SpEnMmjWLgQMHAjBw4EDmzJnD6NGj+eSTT2p02w2ie2fcpeNoe0pb8ory\n2JO/h/2F+zlQdACAA0UH2F+4nz35e8gryqPtKW0Ze+nYKq1/yZIlfP7553z44Ye8/vrrfPfddwD8\n7Gc/Y+bMmWRmZnL77bdz3333ATB8+HDmzp3L2rVr+dnPfkZubu5x1/+Pf/yD1157jccff5y0tDTe\neustXnzxRaZMmVLy/saN4yc/+QkrVqzgzjvvZOLEiQAUFBRwwQUXsHz5cm644QZ+97vf0adPHwYN\nGsRzzz2nwBepoz777DP69u17RFvfvn357LPPanzbDSL0u8Z15S+/+At3XXgXLWJbVDpP88bNSb0w\nlY/u/oiucV2rtP5PP/2U5ORkzIyYmBh69erFzp07iY2N5eyzzwbgmmuuISsri507d9K0adOy9uuv\nv56EhITjrr979+7ExsYSHx9Pz549iY2N5fTTT2fv3r0AZGdn89JLLzFgwAAmT55MTk4OUHIu/NVX\nXw2UfPJfOr+I1G3nnXceq1evPqJt9erVnHfeeTW+7QYR+gBNGzfl+Wue5+Gkh2nZpOUR01rGtmR8\n3/FMvWYqTWKaVHndPXr04P3336e4uJj8/HwyMjKIi4ujoKCArVtLBrZbunQpPXv2pE2bNhQWFrJl\nyxYAVq1axWeffcapp57Kt99+C8D69evZvn17lbb/6KOPkpmZyXvvvce4ceOOO7+ZUVioaxRE6qq0\ntDRSUlLIyMigqKiIjIwMUlJSSEtLq/FtN4g+/fLW5Kxhf+F+ABpbYw75IfYX7efDnA9PeJ1XXXUV\nmZmZ9O7dm9NOO41u3bphZvz2t7/l9ttvp1GjRpxyyinMmDEDgPnz5zNq1CjMjDZt2jBnzhwSEhJ4\n4YUXSE5O5kc/+hFnnnlmxNufMmUKqampPPnkkxQXF4c9jat///6MGTOGu+++m3vvvfeE37eI1Ixh\nw4YBMHr0aD777DPOO+88Jk2aVNZek8zda3wjVZGYmOgVx9Mv3Snh5BXlcdqvT6PgcAHNGzdn8DmD\n+eOWP3Lw0EGaxjRl93/tpnls85oqXSoR6fdOpC575plnWLx4MQcPHuRHP/oR06ZNo0mTqvca1CQz\n2+juieHmazDdOwDLPl9GweEC2rdsz4qRK3jtx6+xYuQK2rdsT8HhApZ9sSzaJYpIPbNz50727t3L\nBx98wJ///Gfy8vJIT6+/F302qNDftncbQ7sNZcs9W+jVoRcAvTr0YvM9mxnabShb90R0YxkRkTJt\n2rRh0qRJmBn79+9n3759dO/ePdplnbCI+vTNbCgwDogBMt19bLlpjYDfAJcArYC33X18aNoPgf8H\nNAVygdvdfXe1voNyxlwyhjGMOar9e02/x2s/fq2mNisidczIkSPZvHkzzZo1A+CBBx7g+uuvP6l1\n3nbbbbz77rs89NBDnHvuudVRZlSEDX0zSwAmAr2BfcCrZnazuy8KzXI28I27J5lZDLDSzHoBWcCr\nwDB3zzazXwITgNE18UZEREpt27aNzMzMstCvDgsWLCAvL4/hw4czb948Ro4cWW3rrk2RdO8MAha5\n+14v+dR3OnBD6UR33+Luk0MvTwMOA18BXYHd7p4dmjYTuLa6ChcROZY9e/bwi1/8gv79+3PPPfeQ\nl5cX8bIVx8R5+umnmTdvHgAtWrSga9eu7Nmzp6ZKr3GRhH4csKPc6+1A24ozmVkm8Akw091zKy7n\n7oUc4z8LM0s1sywzywp39aqISDiJiYlMnDiRlStXEh8fX3YVezilY+JMnTqV/Px8pk6dyvTp03n5\n5ZdJTEykX79+fPXVV9x11101/A5qkLsf9wGkAE+Wez0QePkY834fWAMMAM4EVpeb1hT4PNz2Lrro\nIq/or3/961FttenLL7/0iy++OKo1lDp06JCvWbMm2mVELNrfO5FPP/3UL7vssojm7datmy9fvvyI\ntuXLl3u3bt1qorRqBWR5mHx194iO9JcAN5pZ6Y0a7wDKzlcysyvM7LrQH5DdwFagtbt/AbQ0s9KP\nuYcD75zoH6eqWLAAOneGRo1KnhcsqI2t1o6vv/6a8ePHR7sMkTqhst/1gwcP8sgjj5Rdlf7OO+9w\n4YUXRrS+aI6JU1vChr67bweeouQD2nXAt+6+yMwyzawdkA0MN7P1ZrYW2AW8GVp8JPCSmX0AXA88\nWhNvorwFCyA1FbZuBfeS59TU6gv+e++9l2effRaAN998k4svvpikpCSefvpp3J0LLriAf/7zn6Fa\nFhx1f8vJkydzySWXMGDAADZt2gTAzJkz6d27N3369CnrO5w7dy6jRo1i8ODB/PCHP+Tpp58G4LHH\nHiM7O5sBAwawY8cO1q5dS1JSEv369WP06JLPyL/66iv69+/PyJEjufTSSxkyZAjFxcUAzJkzh169\nepGYmMjjjz8OwI4dO7j22mtJTk5m8ODB7Nq1C3fnpz/9Kf369WPIkCH861//qp4dKFJNjvW7/vvf\nN6dNmzb07t2b5ORkNm7cyKOPRhY90RwTp9ZE8u9AbT5OtnsnIcG95EfgyEdCQsSrOEpp985TTz3l\nv/71r93dfffu3Z6QkOC7du1yd/cbb7zRN23a5C+++KI/+eST7u5+1VVX+d///vcj1pWUlOQ7d+70\n3bt3+969e33z5s3evXt3P3jwoBcVFXlSUpJv377d58yZ44mJiV5QUOAHDx709u3bl9WSnJxctr7/\n+I//KNvGfffd53/4wx/8yy+/9JYtW/rWrVvd3f2yyy7zTZs2+ZYtW7xHjx6el5fn7u7Tpk3zgoIC\nv+2223zevHnu7r548WIfM2aM79692/v16+dFRUW+bds2Ly4uPqF9p+4dqSk18bv+yiuveJcuXXz5\n8uVeWFjoy5cv9y5duvgrr7xSXWXXGCLs3mlwY+9s21a19kh9+umnHDhwoGx8nc8//5wDBw5w0003\nAbBv3z62bNnC7bffzoABAxg5ciSNGjXirLPOOmI98+fP51e/+hXFxcU8/PDDfPzxx+zZs4dBgwaV\nrefzzz8H4PLLLy+71LtRo6P/Kdu5cyf/+7//y5133glAXl4eHTp0oGfPnnTr1o1OnToB/x6B84sv\nvqB///40b14yFMXPf/5zoGQUz61btzJ79myKi4tp27YtrVu3ZuLEidx333106NCBhx56iJiYmJPb\niSLVqCZ+16M5Jk5taXCh36lTyb95lbWfjPPOO4+33nqLa6+9lqVLl3LmmWdyxhln8Pbbb3PKKaew\nefNmWrduTYsWLbj88su55557SE1NPWo9hw4d4je/+Q0rVqzg6aefJjU1la5du7Js2TIaN27Mpk2b\nOOuss8qCv6LyI2jGxcXRpUsXXnvtNU4//XRycnLIz88/5nvo0aMHTz75JHl5ebRo0YKFCxdy7bXX\n0qNHD0aNGsWVV15JQUEBf/7znzl8+DCdOnXi+eef54knnuDtt98+6YtbRKpTTf2uDxs2rEGFfEUN\nahgGgEmToEWFIfVbtChpPxmNGjWibdu2PP7444wYMYLWrVszYcIErrzySvr168cjjzxC06ZNgZIj\n6KysLAYPHnzUembPns2AAQMYO3YsgwcPLrv7Vd++fenXrx//8z//U7aeyrRv354DBw5w+eWXs2fP\nHqZPn84tt9xC//79SU1NJTY29pjLdu3alTFjxtCvXz8uueQS1q9fT6tWrZgyZUrZXbquuOIK9u/f\nz3fffcf48ePp378/f/rTn+jVq9fJ7UCRalbV3/U33niDoUOHlv0HHFiR9AHV5qM6Ttn87W9L+vXM\nSp5/+9sqLX7SXn75ZX/iiSdqd6N1lPr0pSZV5Xc9MzPTc3Nz/fTTT6+t8moVQe3TB7jttpJHNMyZ\nM4e5c+fW61H4ROqLqvyuJycn12wx9USD696JtlGjRrFixQpat24d7VJERI6i0BeRBqkhX6R5Mhpk\n946IBFvphVul46yVXrglOtIXkQYoLe3fgV8qL6+kPeh0pC8iDc7xLtwqLt5R+cSA0JH+SVq1alXZ\n1yNHjmTp0qVRrObfytclEjTHOhU/6KfoQwMN/Yo3QVi4cGGNbWv48OE1tu6TUVfrEqkNNXWRZkPQ\n4EK/spsgpKWlnXTwP/PMM1xyySX06dOHSaGfnMcee4wdO3YwYMAAsrNLbhD2/vvvM2TIEM4//3ze\nffddoPJRLAG6devG9OnTSUlJOWJbGRkZ9O7dm/79+zN37lyAKo2m+eKLL5bVtXTp0mNu/9xzz2XC\nhAlcdtllXHTRReTk5ACwbt06kpKS6NOnD0OHDiUvL4/i4mJ++ctfkpSURP/+/dm4cSNQ+aihItF2\n220wYwYkJIBZyfOMGSd2/c7EiRMZMGBAtdcYNZFcwVWbj5O9IrcmboLw/vvv+5VXXumHDh3yw4cP\n+3XXXefvvPOOu7snlBvSb8SIET569Gh3d1+5cqVff/317u6VjmLp7n7OOeeUtZc3btw4f+utt/zw\n4cP+9ddfu3vVRtOsWNextt+5c2f//e9/7+7uEyZM8ClTpri7+9lnn+1/+9vf3N09PT3dv/rqK3/p\npZd81KhR7u7+z3/+0/v06ePuR48aWpGuyJX6bMOGDT5q1KgjRratqwjqFbk1cROETZs2cdVVV5WN\nMjlo0CCysrLKRsYs75prrgH+PbIlVD6KJUB+fj433HDDUet47LHHeO6551iyZAl33nknzZo1q9Jo\nmhUda/vuztVXX122bE5ODjt37qRp06acffbZAGWDrGVnZ7Nhw4ayI55du3ZRWFh41KihIg3FwYMH\nuf/++1m0aBFDhw6NdjnVpsGFfulNEAYOHFjWdrI3QejZsyfPPfccDzzwAADvvvtu2T0yi4qKwi5f\n2SiWpUqHTi4vNzeX8ePHU1hYyH/+53+ycuXKKo2mCf8ejbNJkybH3X5Fbdq0obCwkC1btnDOOeew\natUq2rRpQ48ePTj11FPLurZWrFhBkyZNjho1dMqUKWH3hzRc27Zt495772Xfvn3ExMQwefJkevTo\nEe2yTsiDDz7ImDFjyg6SGooG16eflpZGSkoKGRkZFBUVkZGRQUpKCmkncYLuFVdcQVJSUtnjwgsv\n5LrrrgPg/PPPp1+/fmzevPmYy1c2iuXxbNiwgYEDB5KcnMyQIUMwsyqNpgklY/H37duXFStWVHn7\n8+fPZ9SoUSQlJfHss89y+umnk5KSwr59+8r2QVZWFnD0qKESbHfffTfPPPMMy5cv55VXXqFDhw7R\nLumELFu2jN27d/PjH/842qVUOyvpCqo7EhMTvTRQSpXezCBSCxcuZNKkSWXLpaWlNejxseuyqn7v\npP7asWMHw4YN46KLLmLdunVccMEFTJ48ueymPXVZxczo0KED7k6L0ClAq1ev5pprruHll1+OcqXH\nZmYb3T0x7IyRdPzX5qM6hlaWukPfu+BYt26dn3rqqf6Xv/zF3d3T0tL8kUceiXJV4UVyi8SG9EFu\ng+veEZHoaN26NT169Cjrw//JT35SdmpvXTZp0iRmzZrFwIEDiY2NZeDAgcyaNavs8yuAzMzM6BVY\nzSIKfTMbambrzWyjmU2uZPpoM/vQzNaa2Qtm1ijU/riZZZtZZuhxwkMeeR3rhpLw9D1r2CpeBLlh\nwwby8vL44osvgJJ+8Z49e0a5yvBq4oy/uizs2TtmlgBMBHoD+4BXzexmd18Umt4NGAwkufthM/sd\ncB3wJtAFGOrufzuZIps1a8auXbuIi4vDzE5mVVJL3J1du3bRrFmzaJciNaD0IshZs2bRt29fVq9e\nTUpKCj//+c+56667KCoqol27dsyaNSvapYZVE2f81WWRnLI5CFjk7nsBzGw6MApYBODun5rZ9e5+\nuNw6D4a+7gTcb2bnA9uA+919Z1WL7NixIzk5OeTm5lZ1UYmiZs2a0bFjx2iXITWgfJcIUNYlMnr0\naD755JMoV1c1pWf8VfwDNqmBjtkQSejHAeWHpdsOHHHiqrvnm1lr4AUg293fC03aAMx394/NbAQw\nFajyaTSxsbF06dKlqouJSA1pSF0ipWf2jR49uuzsnUmTJjXYM/4iCf1vKemmKdUu1FbGzLoDk4FH\n3X1dabu7P1Rutt8Bj1a2gVBffyqgO9WL1AMNrUtk2LBhDTbkK4rkg9wlwI1m1ir0+g6g7K7fZhYP\nPEdJ3/26cu1mZhPN7NRQ09VApSNyufsMd09098T4+PgTeR8iUotq4iJIqR1hj/TdfbuZPQWsNLNC\nYJW7LzKzTOBW4MeU/CeQXu5D1lfcfYaZfQJkmNl+YC9wV028CRGpXUHrEmlI6sUVuSIicnyRXpGr\ni7NERAJEoS8iEiAKfRE5Ya+//jp9+vShX79+ZXdZk7pNoS8iJ+Rf//pX2TDKq1atIiEhgZkzZ0a7\nLAmjwd1ERURqx2mnncbq1avLhto4dOhQvRhGOeh0pC8iJ6xZs2bk5+czZswYDh48yB133BHtkiQM\nhb6InLCcnBxuvPFGBg0axLRp08ruIy11l0JfRCJScSjlefPmMXLkSGbMmMHVV18d7fIkQurTF5Gw\nKhtKediwYRQWFjJ8+PCy+S677DIefbTSIbakjtAVuSISVvfu3Zk6deoRA6xlZGTUy6GUG6pIr8hV\n6ItIWDExMeTn5xMbG1vWVlRURLNmzTh8+PBxlpTaomEYRBqYFStWMGDAgLLHmWeeyX333Vcr2y4d\nSrm8+jyUcpCpT1+knkhOTi67QXdxcTHJyck8+OCDtbLtoN1dqiFT6IvUQ/PmzeOKK66gQ4cOtbI9\nDaXccKhPX6SeOXToEImJiWRmZtK6detolyN1hPr0RRqoN954g6SkJAW+nBCFvkgdVfFiqIULFwIw\nffp0RowYEeXqpL5Sn75IHVTZxVApKSns3buXzZs306tXr2iXKPWU+vRF6iBdDCVVpYuzROoxXQwl\nVaUPckXqMV0MJTUlotA3s6Fmtt7MNprZ5EqmjzazD81srZm9YGaNQu0DQ23rzWy+mTWp7jcg0hCV\nXgyVkZFBUVERGRkZpKSkkJaWFu3SpJ4L+0GumSUAE4HewD7gVTO72d0XhaZ3AwYDSe5+2Mx+B1xn\nZsuBOUBfd88xs2eA0cBRfzRE5Ei6GEpqStg+fTP7OZDg7v8n9PoyYJS7Dy83TzN3zw99/QfgBUr+\nixjh7j8Ntf8HMM/d+x1ve+rTFxGpuurs048DdpR7vR1oW34Gd883s9Zm9gqQ7e7vRbJcuWJTzSzL\nzLJyc3MjKElERE5EJKH/LUeGdbtQWxkz6w68Bvxfd38i0uVKufsMd09098T4+PhIaxcRkSqKJPSX\nADeaWavQ6zuA9NKJZhYPPAcMdfd15Zb7ALjYzNqHXqeUX05ERGpf2A9y3X27mT0FrDSzQmCVuy8y\ns0zgVuDHQBcg3cxKF3vF3WeY2d3AW2ZWAHwOTKiJNyEiIpHRxVkiIg2ALs4SEZGjKPRFRAJEoS8i\nEiAKfRGRAFHoi4gEiEJfRCRAFPoiIgGi0BcRCRCFvohIgCj0RUQCRKEvIhIgCn0RkQBR6IuIBIhC\nX0QkQBT6IiIBotAXEQkQhb6ISIAo9EVEAkShLyISIAp9EZEAUeiLiARIRKFvZkPNbL2ZbTSzyZVM\nv9PMlpjZBxXaR5rZZjPLDD0era7CRUSk6sKGvpklABOBK4FEoKOZ3Vxhtq3Aw0BMhfYuwL3uPiD0\nmFANNYuIyAmK5Eh/ELDI3fe6uwPTgRvKz+Du7wH7Klm2M3Br6Cj/D2bW5WQLFhGRE9c4gnnigB3l\nXm8H2ka4/r8C69w908wGAAuASyvOZGapQCpAp06dIly1iIhUVSRH+t9yZMi3C7WF5e6/dvfM0NeZ\nQGczs0rmm+Huie6eGB8fH8mqRUTkBEQS+kuAG82sVej1HUB6JCs3s/8yszNCXycCX4e6iEREJArC\ndu+4+3YzewpYaWaFwCp3X2RmmcCt7r7jOItvABaZWQFQCAyvjqJFROTEWF078E5MTPSsrKxolyEi\nUq+Y2UZ3Tww3ny7OEhEJEIW+iEiAKPRFRAJEoS8iEiAKfRGRAFHoi4gEiEJfRCRAFPoiIgGi0BcR\nCRCFvohIgCj0RUQCRKEvIhIgCn0RkQBR6IuIBIhCX0QkQBT6IiIBotAXEQkQhb6ISIAo9EVEAkSh\nLyISIBGFvpkNNbP1ZrbRzCZXMv1OM1tiZh9UaP+hma0wsw/N7I9m9v3qKlxERKoubOibWQIwEbgS\nSAQ6mtnNFWbbCjwMxJRbzoBXgTHufgnwDjChmuoWEZETEMmR/iBgkbvvdXcHpgM3lJ/B3d8D9lVY\nriuw292zQ69nAteeZL0iInISIgn9OGBHudfbgbZVXc7dC4HGVapORESqVSSh/y1Hhny7UFuVljOz\npkBhZTOaWaqZZZlZVm5ubgSrFhGRExFJ6C8BbjSzVqHXdwDp4RZy9y+AlmbWPdQ0nJJ+/crmneHu\nie6eGB8fH0FJIiJyIsJ2t7j7djN7ClhpZoXAKndfZGaZwK3uvuM4i48EXjKzYmAXMKIaahYRkRNk\nJZ/N1h2JiYmelZUV7TJEROoVM9vo7onh5tPFWSIiAaLQFxEJEIW+iEiAKPRFRAJEoS8iEiAKfRGR\nAFHoi4gEiEJfRCRAFPoiIgGi0BcRCZAGG/ovvvgivXv3JjExkQkTdO8WERFooKG/ZcsWZs+ezapV\nq1i3bh3r169n+fLl0S5LRCTqGmTof/TRRyQlJdG0aVNiYmK46aabWLZsWbTLEhGJugYZ+j169GDF\nihXs3buXgoICFi1axHfffRftskREoq5BhP6CBdC5MzRqVPKclXUOY8eO5eqrr+bmm2+mV69edOrU\nKdpliohEXb0P/QULIDUVtm4F95Lnu+7K55tvLmTNmjWkp6eTnZ3NLbfcEu1SRUSirt7fqDwtDfLy\njmw7ePAwTzzxJIsXf0WzZs24++67OfPMM6NToIhIHVLvQ3/btspaT+HgwVdYs6a2qxERqdvqfffO\nsbrq1YUvInK0eh/6kyZBixZHtrVoUdIuIiJHqvehf9ttMGMGJCSAWcnzjBkl7SIicqSIQt/MhprZ\nejPbaGaTK5l+b2h6tpmNK9f+eKgtM/RIrc7iS912G3z1FRQXlzwr8EVEKhf2g1wzSwAmAr2BfcCr\nZnazuy8KTU8ChgF9Q4ssN7NMd88CugBD3f1vNVK9iIhUSSRH+oOARe6+190dmA7cUG76dcAcdy90\n90JgNjAkNK0TcL+ZrTCz+WbWpjqLFxGRqokk9OOAHeVebwfaRjh9A/CCuycDfwKmVrYBM0s1sywz\ny8rNzY20dhERqaJIQv9bjgz5dqG2sNPd/SF3/zjU/jtKuoiO4u4z3D3R3RPj4+MjrV1ERKooktBf\nAtxoZq1Cr+8A0stNTwduN7NYM4sBRgBvWomJZnZqaL6rgU3VVbiIiFRd2A9y3X27mT0FrDSzQmCV\nuy8ys0zgVnfPMrM3gfXAIeDV0Ie4mNknQIaZ7Qf2AnfV1BsREZHwrOSz2bojMTHRs7Kyol2GiEi9\nYmYb3T0x3Hz1/uIsERGJnEJfRCRAFPoiIgGi0BcRCRCFvohIgCj0RUQCRKEvIhIgCn0RkQBR6IuI\nBIhCX0QkQBT6IiIBotAXEQkQhb6ISIAo9EVEAkShLyISIAp9EZEAUeiLiASIQr8eKDxcGO0SRKSB\nUOjXcbkHcmn3bDtyD+RGuxQRaQAU+nXc4s2L2Z2/m/Qt6dEuRUQaAIV+HTcne84RzyIiJyOi0Dez\noWa23sw2mtnkSqbfG5qebWZ6qoZLAAAFeElEQVTjyrUPNLO1oWnzzaxJdRbf0O0+uJuN2zcCkPVN\nFnvy90S5IhGp7xqHm8HMEoCJQG9gH/Cqmd3s7otC05OAYUDf0CLLzSwT2AzMAfq6e46ZPQOMBo76\noyElUv+Yyht/fQPHAThUfIjYRrEUHi4ktlEsZ/z3GTRuVPItM4xbzr+F6YOnR7NkEalnIjnSHwQs\ncve97u7AdOCGctOvA+a4e6G7FwKzgSFAErDG3XNC802rsJxUMO7ScbQ9pS15RXnsyd/D/sL9HCg6\nAMCBogPsL9zPnvw95BXl0faUtoy9dGyUKxaR+iaS0I8DdpR7vR1oG8H0cMuVMbNUM8sys6zc3OCe\npdI1rit/+cVfuOvCu2gR26LSeZo3bk7qhal8dPdHdI3rWssVikh9F0nof8uRYd0u1BZuerjlyrj7\nDHdPdPfE+Pj4SOpusJo2bsrz1zzPw0kP07JJyyOmtYxtyfi+45l6zVSaxOjjERGpukhCfwlwo5m1\nCr2+Ayh//mA6cLuZxZpZDDACeBP4ALjYzNqH5kupsJwcx5qcNewv3A9AYyvpx99ftJ8Pcz6MZlki\nUs+FDX133w48Baw0s3XAt+6+yMwyzaydu2dREvLrgQ+BP7p7lrvnA3cDb5nZGqATMLXG3kkDkleU\nR8aXGUBJd85N599E88bNAXj/y/c5WHQwmuWJSD0W9uwdAHdfACyo0Dag3NfPAs9WstyfgItOrsTg\nWfb5MgoOF9C+ZXvSb02nV4debPjnBoa8OoTt+7ez7Itl3HCuPhMXkarTxVl10La92xjabShb7tlC\nrw69AOjVoReb79nM0G5D2bpna5QrFJH6ykrOwqw7EhMTPSsrK9pliIjUK2a20d0Tw82nI30RkQBR\n6IuIBIhCX0QkQBT6IiIBotAXEQkQhb6ISIAo9EVEAkShLyISIAp9EZEAUeiLiASIQl9EJEAU+iIi\nAaLQFxEJEIW+iEiAKPRFRAJEoS8iEiB17iYqZpYL6NZQR2oD7Ix2EXWU9k3ltF+OraHumwR3jw83\nU50LfTmamWVFckecINK+qZz2y7EFfd+oe0dEJEAU+iIiAaLQrx9mRLuAOkz7pnLaL8cW6H2jPn0R\nkQDRkb6ISIAo9OsQMxtqZuvNbKOZTa5k+r2h6dlmNi4aNUZDBPvlTjNbYmYfRKO+aIpg34w2sw/N\nbK2ZvWBmgfmdP96+MbNGZjbZzD4ws4/M7Olo1VnbAvMDUNeZWQIwEbgSSAQ6mtnN5aYnAcOAvkBv\n4AYza/CnnYXbLyFbgYeBmFouL6oi+JnpBgwGkty9DxAPXBeNWmtbBD83ZwPfuHsS8COgv5n1qv1K\na59Cv+4YBCxy971e8kHLdOCGctOvA+a4e6G7FwKzgSFRqLO2hdsvuPt7wL5oFBdlx9037v4pcL27\nHw41NQYO1n6ZURFu32xx99Kj/9OAw8BXtV5lFCj06444YEe519uBtlWY3lAF9X1HIuy+cfd8M2tt\nZq8A2aE/kEEQ0c+NmWUCnwAz3T23dkqLrsbRLkDKfAt0Kfe6Xait/PS2x5neUIXbL0EWdt+YWXdg\nMvCou6+rxdqiLaKfG3cfYGbfB942s23unllL9UWNjvTrjiXAjWbWKvT6DiC93PR04HYzizWzGGAE\n8GYt1xgN4fZLkB1335hZPPAcMDRggQ/h980VZnYdgLvvpuRzoda1XmUUKPTrCHffDjwFrDSzdcC3\n7r7IzDLNrJ27Z1ES8uuBD4E/htoatHD7JcrlRVUE++YnlBztpofaMs0sNZo115YI9k02MDx0ds9a\nYBfBOIjSxVkiIkGiI30RkQBR6IuIBIhCX0QkQBT6IiIBotAXEQkQhb6ISIAo9EVEAkShLyISIP8f\nMlp6fIFeFjgAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jlmXgAH9YlC9",
        "colab_type": "text"
      },
      "source": [
        "## References\n",
        "1. [Kamil Bennani-Smires, Claudiu Musat, Andreaa Hossmann, Michael Baeriswyl, Martin Jaggi. Simple Unsupervised Keyphrase Extraction using Sentence Embeddings. CoNLL 2018, pages 221–229](https://www.aclweb.org/anthology/K18-1022)\n",
        "\n",
        "\n"
      ]
    }
  ]
}