hw8_ignatiev.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import adagram\n",
    "from lxml import html\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.corpus import wordnet as wn\n",
    "from pymorphy2 import MorphAnalyzer\n",
    "from string import punctuation\n",
    "import json, os, re, sys\n",
    "from collections import Counter\n",
    "import numpy as np\n",
    "from razdel import tokenize as razdel_tokenize\n",
    "from matplotlib import pyplot as plt\n",
    "import gensim\n",
    "import pandas as pd\n",
    "from sklearn.metrics import adjusted_rand_score\n",
    "import tqdm\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "\n",
    "%matplotlib inline\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "morph = MorphAnalyzer()\n",
    "punct = punctuation+'«»—…“”*№–'\n",
    "stops = set(stopwords.words('russian'))\n",
    "\n",
    "def normalize(text):\n",
    "    \n",
    "    words = [token.text.strip(punct) for token in list(razdel_tokenize(text))]\n",
    "    words = [morph.parse(word)[0].normal_form for word in words if word and word not in stops]\n",
    "\n",
    "    return words\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\ruthenian8\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping corpora\\wordnet.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# запустите если не установлен ворднет\n",
    "import nltk\n",
    "nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Домашнее задание. Реализовать алгоритм Леска и проверить его на реальном датасете"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lesk( word, sentence ):\n",
    "    \"\"\"Ваш код тут\"\"\"\n",
    "    bestsense = 0\n",
    "    maxoverlap = 0 \n",
    "    orig_set = set(sentence)\n",
    "    wn_sets = []\n",
    "    for i, syns in enumerate(wn.synsets(word)):\n",
    "        wn_sets.append((i,\n",
    "                        set(re.findall(r\"[a-zA-Z]+\", syns.definition()))))\n",
    "         \n",
    "    wn_sorted = sorted(wn_sets,\n",
    "                       key=lambda x: len(x[1].intersection(orig_set)),\n",
    "                       reverse=True)\n",
    "    bestsense = wn_sorted[0][0]\n",
    "    return bestsense\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "class leskMeasure():\n",
    "    def __init__(self, subcorpus):\n",
    "        self.subcorpus = subcorpus\n",
    "        self.all = 0\n",
    "        for sent in subcorpus:\n",
    "            for word in sent:\n",
    "                if word[0] != \"\":\n",
    "                    self.all += 1\n",
    "                    \n",
    "    @staticmethod\n",
    "    def lesk( word, sentence ):\n",
    "        \"\"\"Ваш код тут\"\"\"\n",
    "        bestsense = 0\n",
    "        maxoverlap = 0 \n",
    "        orig_set = set(sentence)\n",
    "        wn_sets = []\n",
    "        for i, syns in enumerate(wn.synsets(word)):\n",
    "            wn_sets.append((i,\n",
    "                            set(re.findall(r\"[a-zA-Z]+\", syns.definition()))))\n",
    "\n",
    "        wn_sorted = sorted(wn_sets,\n",
    "                           key=lambda x: len(x[1].intersection(orig_set)),\n",
    "                           reverse=True)\n",
    "        bestsense = wn_sorted[0][0]\n",
    "        return bestsense                    \n",
    "                    \n",
    "    @staticmethod\n",
    "    def process_sentence(sent):\n",
    "        trues = 0\n",
    "        full_sentence = [x[1] for x in sent] \n",
    "        for item in sent:\n",
    "            if item[0] == \"\":\n",
    "                continue\n",
    "            trues += leskMeasure.process_word(item, full_sentence)\n",
    "        return trues\n",
    "    \n",
    "    @staticmethod\n",
    "    def process_word(item, sent):\n",
    "        if wn.synsets(item[1])[leskMeasure.lesk(item[1], sent)] == \\\n",
    "        wn.lemma_from_key(item[0]).synset():\n",
    "            return 1\n",
    "        return 0\n",
    "    \n",
    "    def compute_accuracy(self):\n",
    "        true_counter = 0\n",
    "        for sent in self.subcorpus:\n",
    "            sent_true_count = self.process_sentence(sent)\n",
    "            true_counter += sent_true_count\n",
    "        return true_counter / self.all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus_wsd = []\n",
    "corpus = open('corpus_wsd_50k.txt').read().split('\\n\\n')\n",
    "for sent in corpus:\n",
    "    corpus_wsd.append([s.split('\\t') for s in sent.split('\\n')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.36496771217712176"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subcorpus = corpus_wsd[:1000]\n",
    "that = leskMeasure(subcorpus)\n",
    "that.compute_accuracy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.37930832356389216"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subcorpus = corpus_wsd[1000:3000]\n",
    "that = leskMeasure(subcorpus)\n",
    "that.compute_accuracy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3604372722540344"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subcorpus = corpus_wsd[3000:5000]\n",
    "that = leskMeasure(subcorpus)\n",
    "that.compute_accuracy()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}