HW7_Ignatiev.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Задание 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Подготовка данных"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting russian-tagsets\n",
      "  Downloading russian-tagsets-0.6.tar.gz (23 kB)\n",
      "Building wheels for collected packages: russian-tagsets\n",
      "  Building wheel for russian-tagsets (setup.py): started\n",
      "  Building wheel for russian-tagsets (setup.py): finished with status 'done'\n",
      "  Created wheel for russian-tagsets: filename=russian_tagsets-0.6-py3-none-any.whl size=24640 sha256=851f3ee5a0bf799a0f0716083a6bf434c8c2637e3f90aa9fcd6db74640ab50d1\n",
      "  Stored in directory: c:\\users\\ruthenian8\\appdata\\local\\pip\\cache\\wheels\\12\\2e\\54\\71c28ef06e79d9bdd7843ad80473900615056abb3261544039\n",
      "Successfully built russian-tagsets\n",
      "Installing collected packages: russian-tagsets\n",
      "Successfully installed russian-tagsets-0.6\n"
     ]
    }
   ],
   "source": [
    "!pip install russian-tagsets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "from russian_tagsets import converters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DF9IS6tnO4UQ",
    "outputId": "4449edf7-8354-40a9-81f0-61a7fd7b7cb8"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\ruthenian8\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from lxml import html\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn.decomposition import TruncatedSVD, NMF, PCA\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.metrics.pairwise import cosine_distances\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import gensim\n",
    "import numpy as np\n",
    "from sklearn.cluster import MiniBatchKMeans\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report\n",
    "from collections import Counter,defaultdict\n",
    "from string import punctuation\n",
    "from razdel import tokenize as razdel_tokenize\n",
    "import os\n",
    "import lxml\n",
    "from bs4 import BeautifulSoup\n",
    "from nltk.corpus import stopwords\n",
    "from pymorphy2 import MorphAnalyzer\n",
    "%matplotlib inline\n",
    "\n",
    "morph = MorphAnalyzer()\n",
    "punct = punctuation+'«»—…“”*№–'\n",
    "import nltk\n",
    "nltk.download(\"stopwords\")\n",
    "stops = set(stopwords.words('russian'))\n",
    "\n",
    "def normalize(text):\n",
    "    \n",
    "    words = [word.strip(punct) for word in text.lower().split()]\n",
    "    words = [morph.parse(word)[0].normal_form for word in words if word and word not in stops]\n",
    "\n",
    "    return ' '.join(words)\n",
    "\n",
    "def tokenize(text):\n",
    "    \n",
    "    \n",
    "    tokens = [token.text for token in list(razdel_tokenize(text))]\n",
    "    tokens = [token for token in tokens if token.isalnum()]\n",
    "\n",
    "    return ' '.join(tokens)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Fs6sslndPb75",
    "outputId": "afd17176-4a47-4f48-a484-e386fd0f8f78"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2021-02-07 18:32:44--  http://vectors.nlpl.eu/repository/20/180.zip\n",
      "Resolving vectors.nlpl.eu (vectors.nlpl.eu)... 129.240.189.181\n",
      "Connecting to vectors.nlpl.eu (vectors.nlpl.eu)|129.240.189.181|:80... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 484452317 (462M) [application/zip]\n",
      "Saving to: ‘180.zip’\n",
      "\n",
      "180.zip             100%[===================>] 462.01M  23.3MB/s    in 21s     \n",
      "\n",
      "2021-02-07 18:33:06 (22.3 MB/s) - ‘180.zip’ saved [484452317/484452317]\n",
      "\n",
      "Archive:  180.zip\n",
      "  inflating: meta.json               \n",
      "  inflating: model.bin               \n",
      "  inflating: model.txt               \n",
      "  inflating: README                  \n"
     ]
    }
   ],
   "source": [
    "!wget \"http://vectors.nlpl.eu/repository/20/180.zip\"\n",
    "!unzip \"180.zip\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "zA7floOzO4Ut"
   },
   "outputs": [],
   "source": [
    "import gensim\n",
    "model = gensim.models.KeyedVectors.load_word2vec_format('model.bin', binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_ud = converters.converter('opencorpora-int', \"ud14\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "id": "p_i4AqWgO4Ut"
   },
   "outputs": [],
   "source": [
    "def gud_normalize(text):\n",
    "    words = text.split()\n",
    "    tokens = []\n",
    "    for word in words:\n",
    "        analysis = morph.parse(word)[0]\n",
    "        lemma = analysis.normal_form\n",
    "        tag = analysis.tag\n",
    "            \n",
    "        if tag._is_unknown() == False:\n",
    "            pos = to_ud(str(tag.POS))\n",
    "            pos = pos[:pos.find(\" \")]\n",
    "        else:\n",
    "            pos = 'UNKN'            \n",
    "        tokens.append(lemma+'_'+pos)\n",
    "    return tokens\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "teYpg4gwQMW3",
    "outputId": "884533cb-62ab-415f-935e-d03deb825288"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Archive:  paraphraser.zip\n",
      "  inflating: LICENSE                 \n",
      "  inflating: corpus.xml              \n",
      "  inflating: paraphrases.xml         \n"
     ]
    }
   ],
   "source": [
    "!unzip \"paraphraser.zip\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "XZgpsgMiTcoq"
   },
   "outputs": [],
   "source": [
    "with open(\"paraphrases.xml\", \"r\", encoding=\"UTF-8\") as file:\n",
    "    corp = file.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "OZPhlxg_T9rV"
   },
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(corp, \"xml\")\n",
    "pars = soup.find_all(\"paraphrase\")\n",
    "dicts = []\n",
    "for index in range(len(pars)):\n",
    "    parent = pars[index]\n",
    "    mapp = {}\n",
    "    mapp[\"text_1\"] = parent.find(name=\"value\", attrs={\"name\": \"text_1\"}).text.lower()\n",
    "    mapp[\"text_2\"] = parent.find(name=\"value\", attrs={\"name\": \"text_2\"}).text.lower()\n",
    "    mapp[\"class\"] = parent.find(name=\"value\", attrs={\"name\": \"class\"}).text\n",
    "    dicts.append(mapp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 202
    },
    "id": "d55eVQ2yUb8p",
    "outputId": "b955b6b6-f77d-431c-cbdf-0e6195480295"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_1</th>\n",
       "      <th>text_2</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>полицейским разрешат стрелять на поражение по ...</td>\n",
       "      <td>полиции могут разрешить стрелять по хулиганам ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>право полицейских на проникновение в жилище ре...</td>\n",
       "      <td>правила внесудебного проникновения полицейских...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>президент египта ввел чрезвычайное положение в...</td>\n",
       "      <td>власти египта угрожают ввести в стране чрезвыч...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>вернувшихся из сирии россиян волнует вопрос тр...</td>\n",
       "      <td>самолеты мчс вывезут россиян из разрушенной си...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>в москву из сирии вернулись 2 самолета мчс с р...</td>\n",
       "      <td>самолеты мчс вывезут россиян из разрушенной си...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              text_1  \\\n",
       "0  полицейским разрешат стрелять на поражение по ...   \n",
       "1  право полицейских на проникновение в жилище ре...   \n",
       "2  президент египта ввел чрезвычайное положение в...   \n",
       "3  вернувшихся из сирии россиян волнует вопрос тр...   \n",
       "4  в москву из сирии вернулись 2 самолета мчс с р...   \n",
       "\n",
       "                                              text_2 class  \n",
       "0  полиции могут разрешить стрелять по хулиганам ...     0  \n",
       "1  правила внесудебного проникновения полицейских...     0  \n",
       "2  власти египта угрожают ввести в стране чрезвыч...     0  \n",
       "3  самолеты мчс вывезут россиян из разрушенной си...    -1  \n",
       "4  самолеты мчс вывезут россиян из разрушенной си...     0  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paraphrases = pd.DataFrame.from_dict(dicts)\n",
    "paraphrases.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "gE9xzov3O4UR"
   },
   "outputs": [],
   "source": [
    "data = open('wiki_data.txt', encoding=\"utf-8\").read().splitlines()\n",
    "\n",
    "data_norm = [normalize(tokenize(text)) for text in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "e_ZcbiqwO4US"
   },
   "outputs": [],
   "source": [
    "data_norm = [text for text in data_norm if text]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Обучение векторов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2vec = gensim.models.Word2Vec([text.split() for text in data_norm], size=300, sg=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "veri_gud_markers = {\"NOUN\", \"VERB\", \"ADJ\", \"ADV\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mean_vector_custom(string, model):\n",
    "    words = string.split()\n",
    "    init = np.zeros(300)\n",
    "    for word in words:\n",
    "        try:\n",
    "            vec = model[word]\n",
    "            init += vec\n",
    "        except:\n",
    "            continue\n",
    "    return init / len(words)\n",
    "            \n",
    "def get_mean_vector_pretrained(string, model):\n",
    "    words = gud_normalize(string)\n",
    "    init = np.zeros(300)\n",
    "    for word in words:\n",
    "        try:\n",
    "            vec = model[word]\n",
    "            gram_marker = word[word.find(\"_\")+1:]\n",
    "            if gram_marker in veri_gud_markers:\n",
    "                init += vec * 0.8\n",
    "            else:\n",
    "                init += vec * 0.2\n",
    "        except:\n",
    "            continue\n",
    "    return init / len(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "paraphrases[\"custom\"] = np.zeros(paraphrases.shape[0])\n",
    "paraphrases[\"pretrained\"] = np.zeros(paraphrases.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "custom_first = []\n",
    "custom_second = []\n",
    "pretrained_first = []\n",
    "pretrained_second = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                         | 0/7227 [00:00<?, ?it/s]<ipython-input-124-1b59c036ecfc>:6: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  vec = model[word]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████| 7227/7227 [00:09<00:00, 728.60it/s]\n"
     ]
    }
   ],
   "source": [
    "for index in tqdm(range(paraphrases.shape[0])):\n",
    "    t1 = paraphrases.loc[index, \"text_1\"]\n",
    "    t2 = paraphrases.loc[index, \"text_2\"]\n",
    "    custom_first.append(get_mean_vector_custom(t1, word2vec))\n",
    "    custom_second.append(get_mean_vector_custom(t2, word2vec))\n",
    "                                                       \n",
    "    pretrained_first.append(get_mean_vector_pretrained(t1, model))\n",
    "    pretrained_second.append(get_mean_vector_pretrained(t2, model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "paraphrases[\"custom\"] = [np.concatenate([a, b]) for a, b in zip(custom_first, custom_second)]\n",
    "paraphrases[\"pretrained\"] = [np.concatenate([a, b]) for a, b in zip(pretrained_first, pretrained_second)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Оценка показателей"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_custom = np.array(paraphrases[\"custom\"].to_list())\n",
    "X_pretrained = np.array(paraphrases[\"pretrained\"].to_list())\n",
    "y = [int(i) for i in paraphrases[\"class\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Показатели для кастомных векторов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.42350141, 0.42715905, 0.41458905, 0.42439358, 0.41451315])"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_1 = LogisticRegression(class_weight='balanced', C=1, n_jobs=3)\n",
    "log_2 = LogisticRegression(class_weight='balanced', C=10, n_jobs=3)\n",
    "cross_val_score(log_1, X_custom, y, scoring=\"f1_macro\", cv=StratifiedKFold(n_splits=5, shuffle=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Для готовых векторов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.41686953, 0.42845478, 0.43880293, 0.42791729, 0.39506934])"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(log_2, X_pretrained, y, scoring=\"f1_macro\", cv=StratifiedKFold(n_splits=5, shuffle=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Задание 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Обучение векторов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "fasttext = gensim.models.FastText([text.split() for text in data_norm], size=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-124-1b59c036ecfc>:6: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  vec = model[word]\n"
     ]
    }
   ],
   "source": [
    "ft_first = [get_mean_vector_custom(i, fasttext) for i in paraphrases[\"text_1\"]]\n",
    "ft_second = [get_mean_vector_custom(i, fasttext) for i in paraphrases[\"text_2\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = CountVectorizer(min_df=4, max_df=0.4, max_features=6000)\n",
    "vocab = np.concatenate((paraphrases[\"text_1\"], paraphrases[\"text_2\"])).tolist()\n",
    "for index in range(len(vocab)):\n",
    "    vocab[index] = normalize(vocab[index])\n",
    "X_matrix = cv.fit_transform(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "id2word = {i:w for i,w in enumerate(cv.get_feature_names())}\n",
    "word2id = {w:i for i,w in id2word.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TruncatedSVD(n_components=300)"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svd = TruncatedSVD(300)\n",
    "svd.fit(X_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NMF(n_components=300)"
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nmf = NMF(300)\n",
    "nmf.fit(X_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "id2vec_nmf = nmf.components_.T\n",
    "id2vec_svd = svd.components_.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean_for_dim_reduction(string, id2vec):\n",
    "    words = string.split()\n",
    "    init = np.zeros(300)\n",
    "    len_count = 0\n",
    "    for word in words:\n",
    "        lemma = morph.parse(word)[0].normal_form\n",
    "        try:\n",
    "            vec = id2vec[word2id[lemma]]\n",
    "            init += vec\n",
    "            len_count += 1\n",
    "        except:\n",
    "            continue\n",
    "    if len_count == 0:\n",
    "        return init\n",
    "    return init / len_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "svd_first = [mean_for_dim_reduction(i, id2vec_svd) for i in paraphrases[\"text_1\"]]\n",
    "svd_second = [mean_for_dim_reduction(i, id2vec_svd) for i in paraphrases[\"text_2\"]]\n",
    "nmf_first = [mean_for_dim_reduction(i, id2vec_nmf) for i in paraphrases[\"text_1\"]]\n",
    "nmf_second = [mean_for_dim_reduction(i, id2vec_nmf) for i in paraphrases[\"text_2\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "cs_custom = [cosine_similarity(a.reshape(1, -1), b.reshape(1, -1)) for a, b in zip(custom_first, custom_second)]\n",
    "cs_pr = [cosine_similarity(a.reshape(1, -1), b.reshape(1, -1)) for a, b in zip(pretrained_first, pretrained_second)]\n",
    "cs_ft = [cosine_similarity(a.reshape(1, -1), b.reshape(1, -1)) for a, b in zip(ft_first, ft_second)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "cs_svd = [cosine_similarity(a.reshape(1, -1), b.reshape(1, -1)) for a, b in zip(svd_first, svd_second)]\n",
    "cs_nmf = [cosine_similarity(a.reshape(1, -1), b.reshape(1, -1)) for a, b in zip(nmf_first, nmf_second)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "paraphrases[\"cs_custom\"] = cs_custom\n",
    "paraphrases[\"cs_pr\"] = cs_pr\n",
    "paraphrases[\"cs_ft\"] = cs_ft\n",
    "paraphrases[\"cs_svd\"] = cs_svd\n",
    "paraphrases[\"cs_nmf\"] = cs_nmf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Оценка показателей"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_num = paraphrases[[\"cs_custom\", \"cs_pr\", \"cs_ft\", \"cs_svd\", \"cs_nmf\"]]\n",
    "y_num = paraphrases[\"class\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.47164592, 0.48893499, 0.50519031, 0.49273356, 0.47958478])"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_3 = LogisticRegression(l1_ratio=0.2, class_weight='balanced', solver='saga', penalty=\"elasticnet\", C=10, max_iter=130, n_jobs=3)\n",
    "cross_val_score(log_3, X_num, y_num, scoring=\"f1_micro\", cv=StratifiedKFold(n_splits=5, shuffle=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Корректировки в методах векторизации"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2vec = gensim.models.Word2Vec([text.split() for text in data_norm], window=6, min_count=3, size=300, sg=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                         | 0/7227 [00:00<?, ?it/s]<ipython-input-124-1b59c036ecfc>:6: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  vec = model[word]\n",
      "100%|████████████████████████████████████████████████████████████████████████████| 7227/7227 [00:01<00:00, 5677.17it/s]\n"
     ]
    }
   ],
   "source": [
    "custom_first = []\n",
    "custom_second = []\n",
    "for index in tqdm(range(paraphrases.shape[0])):\n",
    "    t1 = paraphrases.loc[index, \"text_1\"]\n",
    "    t2 = paraphrases.loc[index, \"text_2\"]\n",
    "    custom_first.append(get_mean_vector_custom(t1, word2vec))\n",
    "    custom_second.append(get_mean_vector_custom(t2, word2vec))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [],
   "source": [
    "cs_custom = [cosine_similarity(a.reshape(1, -1), b.reshape(1, -1)) for a, b in zip(custom_first, custom_second)]\n",
    "paraphrases[\"cs_custom\"] = cs_custom\n",
    "X_num = paraphrases[[\"cs_custom\", \"cs_pr\", \"cs_ft\", \"cs_svd\", \"cs_nmf\"]]\n",
    "y_num = paraphrases[\"class\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.47164592, 0.5055325 , 0.48442907, 0.48581315, 0.49134948])"
      ]
     },
     "execution_count": 240,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_3 = LogisticRegression(l1_ratio=0.2,\n",
    "                           class_weight='balanced',\n",
    "                           solver='saga',\n",
    "                           penalty=\"elasticnet\",\n",
    "                           C=10, max_iter=120,\n",
    "                           n_jobs=3)\n",
    "cross_val_score(log_3,\n",
    "                X_num,\n",
    "                y_num,\n",
    "                scoring=\"f1_micro\",\n",
    "                cv=StratifiedKFold(n_splits=5,\n",
    "                                   shuffle=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Изменил корректировку весов в зависимости от части речи в обеих функциях\n",
    "# Увеличены веса для важных и неважных слов\n",
    "# Слова вне словаря не учитываются при подсчете среднего\n",
    "#\n",
    "def get_mean_vector_custom(string, model):\n",
    "    words = gud_normalize(string)\n",
    "    init = np.zeros(300)\n",
    "    len_count = 0\n",
    "    for word in words:\n",
    "        try:\n",
    "            chunk = word[:word.find(\"_\")]\n",
    "            vec = model[chunk]\n",
    "            gram_marker = word[word.find(\"_\")+1:]\n",
    "            if gram_marker in veri_gud_markers:\n",
    "                init += vec * 1.2\n",
    "            else:\n",
    "                init += vec * 0.6\n",
    "            len_count += 1\n",
    "        except:\n",
    "            continue\n",
    "    if len_count == 0:\n",
    "        return init\n",
    "    return init / len_count\n",
    "            \n",
    "def get_mean_vector_pretrained(string, model):\n",
    "    words = gud_normalize(string)\n",
    "    init = np.zeros(300)\n",
    "    len_count = 0\n",
    "    for word in words:\n",
    "        try:\n",
    "            vec = model[word]\n",
    "            gram_marker = word[word.find(\"_\")+1:]\n",
    "            if gram_marker in veri_gud_markers:\n",
    "                init += vec * 1.2\n",
    "            else:\n",
    "                init += vec * 0.6\n",
    "            len_count += 1\n",
    "        except:\n",
    "            continue\n",
    "    if len_count == 0:\n",
    "        return init\n",
    "    return init / len_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "custom_first = []\n",
    "custom_second = []\n",
    "pretrained_first = []\n",
    "pretrained_second = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                         | 0/7227 [00:00<?, ?it/s]<ipython-input-247-4a8730930e54>:8: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  vec = model[chunk]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████| 7227/7227 [00:17<00:00, 416.81it/s]\n"
     ]
    }
   ],
   "source": [
    "for index in tqdm(range(paraphrases.shape[0])):\n",
    "    t1 = paraphrases.loc[index, \"text_1\"]\n",
    "    t2 = paraphrases.loc[index, \"text_2\"]\n",
    "    custom_first.append(get_mean_vector_custom(t1, word2vec))\n",
    "    custom_second.append(get_mean_vector_custom(t2, word2vec))\n",
    "                                                       \n",
    "    pretrained_first.append(get_mean_vector_pretrained(t1, model))\n",
    "    pretrained_second.append(get_mean_vector_pretrained(t2, model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "cs_custom = [cosine_similarity(a.reshape(1, -1), b.reshape(1, -1)) for a, b in zip(custom_first, custom_second)]\n",
    "cs_pr = [cosine_similarity(a.reshape(1, -1), b.reshape(1, -1)) for a, b in zip(pretrained_first, pretrained_second)]\n",
    "paraphrases[\"cs_custom\"] = cs_custom\n",
    "paraphrases[\"cs_pr\"] = cs_pr\n",
    "X_num = paraphrases[[\"cs_custom\", \"cs_pr\", \"cs_ft\", \"cs_svd\", \"cs_nmf\"]]\n",
    "y_num = paraphrases[\"class\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Оценка показателей"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.53526971, 0.52558783, 0.52179931, 0.52941176, 0.54256055])"
      ]
     },
     "execution_count": 262,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_3 = LogisticRegression(l1_ratio=0.2,\n",
    "                           class_weight='balanced',\n",
    "                           solver='saga',\n",
    "                           penalty=\"elasticnet\",\n",
    "                           C=10, max_iter=120,\n",
    "                           n_jobs=3)\n",
    "cross_val_score(log_3,\n",
    "                X_num,\n",
    "                y_num,\n",
    "                scoring=\"f1_micro\",\n",
    "                cv=StratifiedKFold(n_splits=5,\n",
    "                                   shuffle=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Комментарий: по сравнению с первоначальной моделью отмечается небольшое улучшение качества (~0.05)."
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "Embeddings.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}