04.topicmodel.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "from gesetz import Gesetz, progressBar\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('..')\n",
    "import collections\n",
    "import statistics\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "%matplotlib inline\n",
    "from typing import List, Dict\n",
    "import re\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Topic model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open(\"pickles/laws_links.pickle\", \"rb\") as fp:   # Unpickling\n",
    "    Gesetz.collected_laws = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize \n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "import string\n",
    "import unicodedata\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def remove_symbols(text):\n",
    "        text = text.replace(\"\\xa0\",\"\")\n",
    "        text = text.replace(\"–\",\"\")\n",
    "        text = text.replace(\"§\",\"\")\n",
    "        text = text.replace(\"'\",\"\")\n",
    "        text = text.replace(\"“\",\"\")\n",
    "        text = text.replace(\"„\",\"\")\n",
    "        text = text.replace(\".\",\" \")\n",
    "        text = text.replace(\"‒\",\"\")\n",
    "        text = text.replace(\"--\",\"\")\n",
    "        text = text.replace(\"+++\",\"\")\n",
    "        return text\n",
    "\n",
    "def remove_numbers(text):\n",
    "    return re.sub('( |\\\\\\n)\\(?\\d+[a-z]?\\.?\\)?( |\\\\\\n)', ' ', text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Take care of stopwords, punctuation\n",
    "from nltk.stem.cistem import Cistem\n",
    "\n",
    "\n",
    "german_stopwords = stopwords.words('german')\n",
    "\n",
    "exclude = set([])\n",
    "exclude.add(\"absatz\")\n",
    "exclude.add(\"artikel\")\n",
    "exclude.add(\"paragraph\")\n",
    "exclude.add(\"gesetz\")\n",
    "\n",
    "\n",
    "def get_lemma(word):\n",
    "    lemma = wordnet.morphy(word)\n",
    "    if lemma is None:\n",
    "        return word\n",
    "    else:\n",
    "        return lemma\n",
    "    \n",
    "def get_lemma2(word):\n",
    "    return WordNetLemmatizer().lemmatize(word)\n",
    "\n",
    "\n",
    "def prepare_content_for_lda(law:Gesetz):\n",
    "    text = law.content\n",
    "    text = remove_numbers(remove_symbols(text.casefold()))\n",
    "    tokens = word_tokenize(text)\n",
    "    stemmer = Cistem()\n",
    "    tokens = [get_lemma2(token) for token in tokens if token not in german_stopwords and token not in set(string.punctuation) and len(token)>1]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "1663/1663 |████████████████████████████████████████████████████████████████████████████████████████████████████| 100.0% (ZweckVG)\n"
    }
   ],
   "source": [
    "corpus_clean = {}\n",
    "for law in progressBar(Gesetz.collected_laws.values()):\n",
    "    corpus_clean[law.name_short] = prepare_content_for_lda(law)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "if len(corpus_clean)>1000:\n",
    "    with open(\"pickles/corpus.pickle\", \"wb\") as fp:   #Pickling\n",
    "        pickle.dump(corpus_clean, fp)\n",
    "else:\n",
    "    print(\"Corpus is to short.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## lets start with the topic modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "from gesetz import Gesetz, progressBar\n",
    "with open(\"pickles/laws_links.pickle\", \"rb\") as fp:   # Unpickling\n",
    "    Gesetz.collected_laws = pickle.load(fp)\n",
    "\n",
    "with open(\"pickles/corpus.pickle\", \"rb\") as fp:   # Unpickling\n",
    "    corpus_load = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "corpus_mirror = [key for key in corpus_load.keys()]\n",
    "corpus_raw = [corpus_load[key] for key in corpus_mirror]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "1663\n"
    }
   ],
   "source": [
    "print(len(corpus_raw))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "['``', 'gesetz', 'rabatte', 'arzneimittel', 'dezember', 'bgbl', '2262', '2275', 'zuletzt', 'artikel', 'gesetzes', 'november', 'bgbl', 'geändert', 'worden', \"''\", 'stand', 'zuletzt', 'geändert', 'art', '11', 'fussnote', 'textnachweis', 'ab', 'wurde', 'artikel', '12', 'bundestag', 'beschlossen', 'gem', 'artikel', 'ab', 'kraft', 'getreten', 'anspruch', 'abschläge', 'pharmazeutischen', 'unternehmer', 'unternehmen', 'privaten', 'krankenversicherung', 'trägern', 'kosten', 'krankheits-', 'pflege-', 'geburtsfällen', 'beamtenrechtlichen', 'vorschriften', 'verschreibungspflichtige', 'arzneimittel', 'deren', 'kosten', 'ganz', 'teilweise', 'erstattet', 'anteil', 'kostentragung', 'abschläge', 'entsprechend', 'absatz', '1a', 'fünften', 'buches', 'sozialgesetzbuch', 'gewähren', 'gilt', 'sonstige', 'träger', 'kosten', 'krankheitsfällen', 'rahmen', 'absicherung', 'krankheitsfall', 'tragen', 'versicherungspflicht', 'absatz', 'satz', 'versicherungsvertragsgesetzes', 'absatz', 'nummer', 'fünften', 'buches', 'sozialgesetzbuch', 'ausgeschlossen', 'ermittlung', 'abschläge', 'satz', 'selbst-', 'eigenbehalte', 'unternehmen', 'privaten', 'krankenversicherung', 'versicherungsnehmern', 'vereinbart', 'beamtenrechtlichen', 'vorschriften', 'vorschriften', 'beruhen', 'berücksichtigen', 'abschläge', 'satz', 'dürfen', 'unternehmen', 'privaten', 'krankenversicherung', 'ausschliesslich', 'vermeidung', 'begrenzung', 'prämienerhöhungen', 'prämienermässigung', 'versichertenbeständen', 'verwendet', 'anspruch', 'ausgleich', 'differenzbetrags', 'erstattungsbetrag', 'tatsächlichem', 'abgabepreis', 'arzneimittel', 'erstattungsbetrag', 'fünften', 'buches', 'sozialgesetzbuch', 'gilt', 'pharmazeutischen', 'unternehmer', 'unternehmen', 'privaten', 'krankenversicherung', 'trägern', 'kosten', 'krankheits-', 'pflege-', 'geburtsfällen', 'beamtenrechtlichen', 'vorschriften', 'differenz', 'erstattungsbetrag', 'tatsächlichen', 'abgabepreis', 'anteil', 'kostentragung', 'auszugleichen', 'sofern', 'tatsächliche', 'abgabepreis', 'zeitpunkt', 'abgabe', 'geltenden', 'erstattungsbetrag', 'liegt', 'satz', 'sowie', 'gelten', 'entsprechend', 'nachweis', 'unternehmen', 'privaten', 'krankenversicherung', 'träger', 'kosten', 'krankheits-', 'pflege-', 'geburtsfällen', 'beamtenrechtlichen', 'vorschriften', 'bilden', 'verband', 'privaten', 'krankenversicherung', 'zentrale', 'stelle', 'einzug', 'abschläge', 'beauftragt', 'nachweis', 'abschlags', 'übermittelt', 'zentrale', 'stelle', 'beauftragte', 'stelle', 'pharmazentralnummer', 'abgegebenen', 'arzneimittels', 'abgabedatum', 'apothekenkennzeichen', 'anteil', 'kostentragung', 'maschinenlesbar', 'pharmazeutischen', 'unternehmer', 'pharmazeutischen', 'unternehmer', 'abschläge', 'innerhalb', 'zehn', 'tagen', 'geltendmachung', 'anspruchs', 'erstatten', 'träger', 'kosten', 'krankheits-', 'pflegeund', 'geburtsfällen', 'beamtenrechtlichen', 'vorschriften', 'verband', 'privaten', 'krankenversicherung', 'wahrnehmung', 'wirtschaftlichen', 'interessen', 'gebildeten', 'spitzenorganisationen', 'pharmazeutischen', 'unternehmer', 'weitere', 'einzelheiten', 'abrechnung', 'zahlungsfrist', 'abweichend', 'gesetz', 'vereinbaren', 'sonstige', 'träger', 'satz', 'berechtigt', 'abrechnung', 'abschläge', 'entweder', 'durchzuführen', 'zentrale', 'stelle', 'angemessener', 'beteiligung', 'kosten', 'durchführen', 'lassen', 'vereinbarungen', 'satz', 'beitreten', 'prüfung', 'treuhänder', 'pharmazeutischen', 'unternehmer', 'begründeten', 'fällen', 'sowie', 'stichproben', 'abrechnung', 'abschläge', 'treuhänder', 'innerhalb', 'jahres', 'ab', 'geltendmachung', 'anspruchs', 'überprüfen', 'lassen', 'hierfür', 'dürfen', 'treuhänder', 'prüfungszweck', 'erforderlichen', 'personenbezogenen', 'daten', 'übermittelt', 'nachweis', 'dürfen', 'reproduktionen', 'digitalisierten', 'verordnungsblättern', 'vorgelegt', 'treuhänder', 'darf', 'übermittelten', 'daten', 'zwecke', 'überprüfung', 'abrechnung', 'abschläge', 'verarbeiten', 'weitere', 'einzelheiten', 'prüfung', 'vereinbarung', 'satz', 'geregelt', 'angaben', 'verordnungsblatt', 'abgabe', 'arzneimitteln', 'abschlagspflicht', 'absatz', '1a', 'fünften', 'buches', 'sozialgesetzbuch', 'unterliegen', 'personen', 'wege', 'sachleistung', 'gesetzlichen', 'krankenversicherung', 'erhalten', 'apotheken', 'neben', 'abgabepreis', 'pharmazentralnummer', 'abgabedatum', 'apothekenkennzeichen', 'eignung', 'verordnungsblatts', 'maschinenlesbarer', 'form', 'übertragen', 'weitere', 'einzelheiten', 'träger', 'kosten', 'krankheits-', 'pflege-', 'geburtsfällen', 'beamtenrechtlichen', 'vorschriften', 'verband', 'privaten', 'krankenversicherung', 'wahrnehmung', 'wirtschaftlichen', 'interessen', 'gebildeten', 'spitzenorganisation', 'apotheker', 'abweichend', 'gesetz', 'vereinbaren', 'insbesondere', 'vereinbart', 'apotheken', 'beleg', 'maschinenlesbarer', 'form', 'pharmazentralnummer', 'abgegebenen', 'arzneimittels', 'kaufpreis', 'abgabedatum', 'ausstellen', 'sonstige', 'träger', 'satz', 'vereinbarung', 'satz', 'beitreten', 'datenübermittlung', 'pharmazeutische', 'unternehmer', 'pharmazeutischen', 'unternehmer', 'verpflichtet', 'erforderlichen', 'preis-', 'produktangaben', 'arzneimittel', 'einschliesslich', 'abschlags', 'verband', 'privaten', 'krankenversicherung', 'antrag', 'träger', 'kosten', 'krankheits-', 'pflege-', 'geburtsfällen', 'beamtenrechtlichen', 'vorschriften', 'sonstige', 'träger', 'satz', 'mehrere', 'benannte', 'stellen', 'maschinell', 'lesbaren', 'datenträgern', 'übermitteln']\n"
    }
   ],
   "source": [
    "print(corpus_raw[100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Importing Gensim\n",
    "import time\n",
    "\n",
    "import gensim\n",
    "from gensim import corpora\n",
    "from gensim.models import LdaMulticore, LdaModel, TfidfModel\n",
    "\n",
    "import pyLDAvis.gensim\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "C:\\Users\\marc\\AppData\\Roaming\\Python\\Python37\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n  and should_run_async(code)\n148207\n147091\n"
    }
   ],
   "source": [
    "# Creating the term dictionary of our courpus, where every unique term is assigned an index. \n",
    "dictionary = corpora.Dictionary(corpus_raw)\n",
    "print(len(dictionary))\n",
    "dictionary.filter_extremes(no_below=0,no_above=0.1,keep_n=150000)\n",
    "print(len(dictionary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "C:\\Users\\marc\\AppData\\Roaming\\Python\\Python37\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n  and should_run_async(code)\n"
    }
   ],
   "source": [
    "# Converting list of documents (corpus) into a matrix which counts the occurence of each word in the dictionary in the specific document\n",
    "corpus = [dictionary.doc2bow(law) for law in corpus_raw]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "C:\\Users\\marc\\AppData\\Roaming\\Python\\Python37\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n  and should_run_async(code)\n"
    }
   ],
   "source": [
    "# Set training parameters.\n",
    "num_topics = 30\n",
    "chunksize = 300\n",
    "passes = 300\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "C:\\Users\\marc\\AppData\\Roaming\\Python\\Python37\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n  and should_run_async(code)\nC:\\Users\\marc\\AppData\\Local\\Programs\\Python\\Python37\\lib\\site-packages\\matplotlib\\__init__.py:107: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working\n  from collections import MutableMapping\nC:\\Users\\marc\\AppData\\Local\\Programs\\Python\\Python37\\lib\\site-packages\\matplotlib\\rcsetup.py:20: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working\n  from collections import Iterable, Mapping\nC:\\Users\\marc\\AppData\\Local\\Programs\\Python\\Python37\\lib\\site-packages\\matplotlib\\rcsetup.py:20: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working\n  from collections import Iterable, Mapping\nC:\\Users\\marc\\AppData\\Local\\Programs\\Python\\Python37\\lib\\site-packages\\matplotlib\\colors.py:53: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working\n  from collections import Sized\n"
    }
   ],
   "source": [
    "import re\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import os\n",
    "\n",
    "import logging\n",
    "logging.basicConfig(filename='gensim.log',\n",
    "                    format=\"%(asctime)s:%(levelname)s:%(message)s\",\n",
    "                    level=logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "C:\\Users\\marc\\AppData\\Roaming\\Python\\Python37\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n  and should_run_async(code)\n"
    }
   ],
   "source": [
    "from gensim.models.callbacks import CoherenceMetric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "C:\\Users\\marc\\AppData\\Roaming\\Python\\Python37\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n  and should_run_async(code)\n"
    }
   ],
   "source": [
    "from gensim.models.callbacks import CallbackAny2Vec\n",
    "class EpochLogger(CallbackAny2Vec):\n",
    "    logger=\"shell\"\n",
    "    init = 0\n",
    "    def __init__(self):\n",
    "\n",
    "        self.epoch = 1\n",
    "    def on_epoch_end(self, model):\n",
    "        print(\"Epoch #{} end\".format(self.epoch))\n",
    "    def get_value(self,topics, model, other_model):\n",
    "        if self.init == 0:\n",
    "            self.init = time.perf_counter()\n",
    "        elapsed = time.perf_counter() - self.init\n",
    "        print(\"Epoch #\"+str(self.epoch)+\" after \"+str(math.floor(elapsed/3600))+\"hrs \"+str(math.floor((elapsed%3600)/60))+ \"min \"+str(math.floor(elapsed%60))+\"sec\")\n",
    "\n",
    "        if self.epoch % 10 ==0:\n",
    "            print(\"saving image\")\n",
    "            p = re.compile(r\"(-*\\d+\\.\\d+) per-word .* (\\d+\\.\\d+) perplexity\")\n",
    "            matches = [p.findall(l) for l in open('gensim.log')]\n",
    "            matches = [m for m in matches if len(m) > 0]\n",
    "            tuples = [t[0] for t in matches]\n",
    "            perplexity = [float(t[1]) for t in tuples]\n",
    "            liklihood = [float(t[0]) for t in tuples]\n",
    "            iter = list(range(0,len(tuples)*10,10))\n",
    "            plt.plot(iter,liklihood,c=\"black\")\n",
    "            plt.ylabel(\"log liklihood\")\n",
    "            plt.xlabel(\"iteration\")\n",
    "            plt.title(\"Topic Model Convergence\")\n",
    "            plt.grid()\n",
    "            plt.savefig(\"training/convergence_likelihood_\"+str(self.epoch)+\".png\")\n",
    "            plt.close()\n",
    "\n",
    "        if self.epoch % 50 ==0 or self.epoch == 10:\n",
    "            print(\"creating pyldavis\")\n",
    "            try:\n",
    "                visualization = pyLDAvis.gensim.prepare(model,corpus,dictionary, mds='mmds',sort_topics=False)\n",
    "                pyLDAvis.save_html(visualization,\"ldavis/vis-\"+str(self.epoch)+\".html\")\n",
    "            except TypeError:\n",
    "                print(\"Oops!  Saving pyLDAvis for epoch \"+self.epoch+\" gave a TypeError...\")\n",
    "            print(\"finished.\")\n",
    "\n",
    "        self.epoch += 1\n",
    "        return 0\n",
    "epoch_logger = EpochLogger()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Running and training LDA model on the documents word_occurence_counting_matrix\n",
    "\n",
    "start = time.perf_counter()\n",
    "ldamodel = LdaModel(\n",
    "    corpus=corpus,\n",
    "    id2word=dictionary,\n",
    "    chunksize=chunksize,\n",
    "    alpha='auto',\n",
    "    eta='auto',\n",
    "    num_topics=num_topics,\n",
    "    passes=passes,\n",
    "    callbacks=[epoch_logger],\n",
    "    random_state=2021,\n",
    ")\n",
    "end = time.perf_counter()\n",
    "\n",
    "sec = end - start\n",
    "\n",
    "print(\"Finished! \"+str(sec/3600))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ldamodel.save(fname=\"savedmodel/model_30_300_fulldict\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## load the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "C:\\Users\\marc\\AppData\\Roaming\\Python\\Python37\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n  and should_run_async(code)\n"
    }
   ],
   "source": [
    "# names of topics\n",
    "ldamodel = LdaModel.load(fname=\"savedmodels/model_30_300_fulldict\")\n",
    "topic_names = {\n",
    "    2:(1,\"GesR\"),\n",
    "    28:(2,\"UmweltR\"),\n",
    "    21:(3,\"ProzessR\"),\n",
    "    14:(4,\"StrafR\"),\n",
    "    16:(5,\"BahnR\"),\n",
    "    12:(6,\"ArbR\"),\n",
    "    26:(7,\"BeamtenR\"),\n",
    "    13:(8,\"MedR\"),\n",
    "    7:(9,\"SteuerR\"),\n",
    "    5:(10,\"VersorgungsR\"),\n",
    "    25:(11,\"SachenR\"),\n",
    "    8:(12,\"OrganR\"),\n",
    "    29:(13,\"VerkehrsR\"),\n",
    "    6:(14,\"FinMarktR\"),\n",
    "    15:(15,\"EntsorgungsR\"),\n",
    "    4:(16,\"KommR\"),\n",
    "    20:(17,\"WahlR\"),\n",
    "    24:(18,\"BauR\"),\n",
    "    23:(19,\"GebührenR\"),\n",
    "    3:(20,\"BerufsR\"),\n",
    "    30:(21,\"InsolvenzR\"),\n",
    "    27:(22,\"StatistikR\"),\n",
    "    17:(23,\"BankR\"),\n",
    "    10:(24,\"PatentR\"),\n",
    "    1:(25,\"ProduktR\"),\n",
    "    18:(26,\"AgrarR\"),\n",
    "    19:(27,\"KriegsfolgenR\"),\n",
    "    11:(28,\"WohnR\"),\n",
    "    9:(29,\"AsylR\"),\n",
    "    22:(30,\"SonderVersR\"),\n",
    "   }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "visualization_unsorted = pyLDAvis.gensim.prepare(ldamodel,corpus,dictionary, mds='mmds',sort_topics=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pyLDAvis.show(visualization_unsorted)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Do inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "[(1, 0.025421666), (7, 0.033902537), (11, 0.61739963), (14, 0.030636057), (15, 0.12550537), (23, 0.033177197), (24, 0.057360116), (25, 0.030003246), (27, 0.025330858), (28, 0.015133855)]\nC:\\Users\\marc\\AppData\\Roaming\\Python\\Python37\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n  and should_run_async(code)\n"
    }
   ],
   "source": [
    "new_doc_bow = dictionary.doc2bow(corpus_raw[830])\n",
    "print(ldamodel.get_document_topics(new_doc_bow))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "C:\\Users\\marc\\AppData\\Roaming\\Python\\Python37\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n  and should_run_async(code)\n"
    }
   ],
   "source": [
    "#categorize the corpus by the retrieved topics\n",
    "speech_topic_matrix = []\n",
    "\n",
    "for law in corpus:\n",
    "    vector = ldamodel[law]\n",
    "    speech_topic_matrix.append(vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "C:\\Users\\marc\\AppData\\Roaming\\Python\\Python37\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n  and should_run_async(code)\n"
    }
   ],
   "source": [
    "end = len(corpus_mirror)\n",
    "#end = 5\n",
    "Gesetz.topic_names = topic_names\n",
    "for index in range(end):\n",
    "    topics = speech_topic_matrix[index]\n",
    "    #print(corpus_mirror[index]+str(topics))\n",
    "    Gesetz.collected_laws[corpus_mirror[index]].set_topics(topics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "{'OrganR': 116, 'BeamtenR': 85, 'VersorgungsR': 158, 'ArbR': 90, 'MedR': 80, 'KommR': 19, 'UmweltR': 219, 'KriegsfolgenR': 22, 'StatistikR': 44, 'BankR': 56, 'GebührenR': 10, 'EntsorgungsR': 21, 'FinMarktR': 16, 'InsolvenzR': 17, 'WohnR': 5, 'GesR': 174, 'BahnR': 66, 'SachenR': 71, 'WahlR': 12, 'ProduktR': 14, 'ProzessR': 106, 'StrafR': 106, 'SteuerR': 70, 'PatentR': 19, 'BauR': 23, 'AgrarR': 12, 'VerkehrsR': 19, 'BerufsR': 10, 'AsylR': 3}\nC:\\Users\\marc\\AppData\\Roaming\\Python\\Python37\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n  and should_run_async(code)\n"
    }
   ],
   "source": [
    "print(Gesetz.get_topic_count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for (num,words) in ldamodel.show_topics(30,20,formatted=False):\n",
    "    topic = topic_names[num+1][1]\n",
    "    if topic in [\"GesR\",\"FinMarktR\",\"OrganR\",\"ArbR\",\"WahlR\",\"ProzessR\",\"SachenR\",\"UmweltR\",\"InsolvenzR\",\"StrafR\"]:\n",
    "        print((topic+\": \").ljust(12)+str([word[0] for word in words]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "C:\\Users\\marc\\AppData\\Roaming\\Python\\Python37\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n  and should_run_async(code)\n"
    }
   ],
   "source": [
    "import pickle\n",
    "with open(\"pickles/laws_links_topics.pickle\", \"wb\") as fp:   #Pickling\n",
    "    pickle.dump(Gesetz.collected_laws, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "C:\\Users\\marc\\AppData\\Roaming\\Python\\Python37\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n  and should_run_async(code)\nGesR:       ['versicherer', 'aktien', 'handelsgesetzbuchs', 'versicherungsnehmer', 'genossenschaft', 'gesellschafter', 'ehegatten', 'kündigung', 'aif', 'verbraucher', 'rechtsträger', 'sache', 'verbindlichkeiten', 'unterabschnitt', 'aktiengesetzes', 'unternehmer', 'vermögensgegenstände', 'handelsregister', 'geschäftsjahr', 'gesellschaften']\nFinMarktR:  ['aufsichtsbehörde', 'institut', 'versicherungsunternehmen', 'bundesbank', 'kunden', 'wertpapierdienstleistungsunternehmen', 'kreditwesengesetzes', 'anleger', 'inländischen', 'instituts', 'handel', 'gruppe', 'wertpapiere', 'risiken', 'institute', 'delegierten', 'finanzinstrumente', 'organisierten', '575/2013', 'aktien']\nOrganR:     ['stiftung', 'vorstand', 'bundestages', 'frau', 'präsident', 'sitzungen', 'mitgliedschaft', 'dienststelle', 'betriebsrat', 'ausschuss', 'stellvertreter', 'amtszeit', 'präsidenten', 'verwaltungsrat', 'gewählt', 'vorsitzende', 'geschäftsordnung', 'stimmen', 'organe', 'vorstandes']\nArbR:       ['arbeitgeber', 'beiträge', 'arbeiten', 'arbeitgebers', 'tarifvertrag', 'arbeitszeit', 'wirtschaftsprüferkammer', 'ausgaben', 'betriebe', 'arbeitnehmers', 'ausbildungsstätte', 'betrieblichen', 'auszubildenden', 'fonds', 'beitrag', 'angestellte', 'auszubildende', 'fünften', 'finanzierung', 'einnahmen']\nStrafR:     ['tat', 'straftaten', 'strafe', 'munition', 'personenbezogenen', 'strafgesetzbuches', 'schusswaffen', 'täter', 'waffen', 'gespeicherten', 'strafprozessordnung', 'schweren', 'gespeichert', 'speicherung', 'anhaltspunkte', 'löschung', 'besitz', 'datenübermittlung', '2016/679', 'registerbehörde']\nWahlR:      ['stimmzettel', 'wahlberechtigten', 'liste', 'unterschrift', 'partei', 'bewerber', 'niederschrift', 'vorschlagslisten', 'beisitzer', 'se', 'vertrauensperson', 'familienname', 'eides', 'vorschlagsliste', 'wahlberechtigte', 'personenbezogenen', 'stellvertreter', 'bitte', 'versammlung', 'stimmen']\nProzessR:   ['verhandlung', 'vollstreckung', 'urteil', 'richter', 'partei', 'zwangsvollstreckung', 'gerichts', 'staatsanwaltschaft', 'sache', 'parteien', 'mündliche', 'geschäftsstelle', 'akten', 'zeugen', 'hauptverhandlung', 'mündlichen', 'berufung', 'oberlandesgericht', 'urteils', 'urkunden']\nSachenR:    ['grundstück', 'grundstücks', 'gemeinde', 'idf', 'gebäude', 'abweichung', 'flächen', 'grundstücken', 'grundstückseigentümer', 'nutzer', 'gvbl', 'enteignung', 'gebäudes', 'gewässer', 'grundbuch', '2010', 'bescheid', 'baulichen', 'bauliche', 'bewilligung']\nUmweltR:    ['stoffe', 'umwelt', 'beförderung', 'untersuchung', 'stoffen', 'angenommen', 'abfälle', 'naturschutz', 'gefahren', 'reaktorsicherheit', 'baugewerbe', 'menschen', 'untersuchungen', 'bord', 'arbeiten', 'schiffes', 'schäden', 'rechtsanwaltskammer', 'baugewerbes', 'bevölkerung']\nInsolvenzR: ['schuldner', 'gläubiger', 'schuldners', 'forderung', 'gläubigers', 'forderungen', 'eröffnung', 'insolvenzverfahrens', 'verwalter', 'insolvenzverwalter', 'befriedigung', 'hypothek', 'zwangsvollstreckung', 'zwangsversteigerung', 'schiff', 'insolvenzverfahren', 'schiffsregister', 'insolvenzordnung', 'plan', 'grundstück']\n"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.7 64-bit",
   "language": "python",
   "name": "python_defaultSpec_1600878640312"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}