Generatingblanks.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "import yake\n",
        "from summarizer import Summarizer\n",
        "from keybert import KeyBERT\n",
        "from nltk.tokenize import sent_tokenize\n",
        "from flashtext import KeywordProcessor\n",
        "\n",
        "def bert_summ(full_text) :\n",
        "  model = Summarizer()\n",
        "  result = model(full_text, min_length=60, max_length = 1000 , ratio = 0.4)\n",
        "  summarized_text = ''.join(result)\n",
        "  return summarized_text\n",
        "\n",
        "def keyword_selection(text) :\n",
        "  kw_extractor = yake.KeywordExtractor()\n",
        "  keywords_yake = kw_extractor.extract_keywords(text)\n",
        "  kw_model = KeyBERT()\n",
        "  keywords_bert = kw_model.extract_keywords(text)\n",
        "  keywords_yake_l = [i[0] for i in keywords_yake]\n",
        "  keywords_bert_l = [i[0] for i in keywords_bert]\n",
        "  keywords = keywords_yake_l + keywords_bert_l \n",
        "  return keywords\n",
        "\n",
        "class SentenceMapping :\n",
        "  def tokenize_sentences(text):\n",
        "      sentences = [sent_tokenize(text)]\n",
        "      sentences = [y for x in sentences for y in x]\n",
        "      # Remove any short sentences less than 20 letters.\n",
        "      sentences = [sentence.strip() for sentence in sentences if len(sentence) > 20]\n",
        "      return sentences\n",
        "      \n",
        "  def get_sentences_for_keyword(keywords, sentences):\n",
        "      keyword_processor = KeywordProcessor()\n",
        "      keyword_sentences = {}\n",
        "      for word in keywords:\n",
        "          keyword_sentences[word] = []\n",
        "          keyword_processor.add_keyword(word)\n",
        "      for sentence in sentences:\n",
        "          keywords_found = keyword_processor.extract_keywords(sentence)\n",
        "          for key in keywords_found:\n",
        "              keyword_sentences[key].append(sentence)\n",
        "      for key in keyword_sentences.keys():\n",
        "              values = keyword_sentences[key]\n",
        "              values = sorted(values, key=len, reverse=True)\n",
        "              keyword_sentences[key] = values\n",
        "      a1 = []\n",
        "      for x, y in keyword_sentences.items():\n",
        "        if len(y) == 0 :\n",
        "          a1.append(x)\n",
        "        elif len(y) > 5 :\n",
        "          a1.append(x)\n",
        "\n",
        "      for i in a1 :\n",
        "        del keyword_sentences[i]\n",
        "      return keyword_sentences"
      ],
      "metadata": {
        "id": "jyebA8KaZ_e9"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}