hw9_1_ignatiev.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from natasha import (\n",
    "    Segmenter,\n",
    "    MorphVocab,\n",
    "    NewsEmbedding,\n",
    "    NewsMorphTagger,\n",
    "    NewsSyntaxParser,\n",
    "    NewsNERTagger,\n",
    "    PER,\n",
    "    NamesExtractor,\n",
    "    Doc,\n",
    "    DatesExtractor,\n",
    "    MoneyExtractor,\n",
    "    AddrExtractor\n",
    ")\n",
    "\n",
    "\n",
    "from ipymarkup import show_span_box_markup\n",
    "\n",
    "segmenter = Segmenter()\n",
    "morph_vocab = MorphVocab()\n",
    "\n",
    "emb = NewsEmbedding()\n",
    "morph_tagger = NewsMorphTagger(emb)\n",
    "syntax_parser = NewsSyntaxParser(emb)\n",
    "ner_tagger = NewsNERTagger(emb)\n",
    "\n",
    "names_extractor = NamesExtractor(morph_vocab)\n",
    "dates_extractor = DatesExtractor(morph_vocab)\n",
    "money_extractor = MoneyExtractor(morph_vocab)\n",
    "addr_extractor = AddrExtractor(morph_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-3-fd0b9af24538>:2: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
      "  pd.set_option('display.max_colwidth', -1)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "pd.set_option('display.max_colwidth', -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from yargy import Parser, rule, or_\n",
    "from yargy.predicates import in_, in_caseless\n",
    "from yargy.tokenizer import MorphTokenizer\n",
    "from yargy.pipelines import morph_pipeline, caseless_pipeline\n",
    "from yargy.interpretation import fact\n",
    "from IPython.display import display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"avito_cars.txt\", encoding=\"UTF-8\") as file:\n",
    "    text = file.read()\n",
    "    car_data = text.splitlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "car_data = pd.Series(car_data, dtype=\"U\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(53726,)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "Mashina = fact(\n",
    "    'Mashina',\n",
    "    ['label', 'model', 'date']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "years = {\n",
    "    \"96\":\"1996\",\n",
    "    \"97\":\"1997\",\n",
    "    \"98\":\"1998\",\n",
    "    \"99\":\"1999\",\n",
    "    \"2000\":\"2000\",\n",
    "    \"1999\":\"1999\",\n",
    "    \"1998\":\"1998\",\n",
    "    \"1997\":\"1997\",    \n",
    "    \"1996\":\"1996\",\n",
    "    \"11\":\"2011\",\n",
    "    \"12\":\"2012\",\n",
    "    \"13\":\"2013\",\n",
    "    \"14\":\"2014\",\n",
    "    \"15\":\"2015\",\n",
    "    \"16\":\"2016\",\n",
    "    \"17\":\"2017\",\n",
    "    \"18\":\"2018\",\n",
    "    \"2004\":\"2004\",\n",
    "    \"2005\":\"2005\",\n",
    "    \"2006\":\"2006\",\n",
    "    \"2007\":\"2007\",\n",
    "    \"2008\":\"2008\",\n",
    "    \"2009\":\"2009\",\n",
    "    \"2011\":\"2011\",\n",
    "    \"2012\":\"2012\",\n",
    "    \"2013\":\"2013\",\n",
    "    \"2014\":\"2014\",\n",
    "    \"2015\":\"2015\",\n",
    "    \"2016\":\"2016\",\n",
    "    \"2017\":\"2017\",\n",
    "    \"2018\":\"2018\"    \n",
    "}\n",
    "vaz_types = {\n",
    "    \"21063\":\"21063\",\n",
    "    \"21074\":\"21074\",\n",
    "    \"2112\":\"2112\",\n",
    "    \"2109\":\"2109\",\n",
    "    \"2106\":\"2106\"\n",
    "}    \n",
    "VAZ = rule(\n",
    "    morph_pipeline([\"ВАЗ\", \"ваз\", \"Ваз\"]).interpretation(Mashina.label.const(\"ГАЗ\")),\n",
    "    caseless_pipeline(vaz_types).interpretation(Mashina.model.normalized().custom(vaz_types.get)).optional(),\n",
    "    caseless_pipeline(years).interpretation(Mashina.date.normalized().custom(years.get)).optional()   \n",
    ")\n",
    "gaz_types = {\n",
    "    \"330202\":\"330202\",\n",
    "    \"322132\":\"322132\"\n",
    "}\n",
    "\n",
    "GAZ = rule(\n",
    "    morph_pipeline([\"ГАЗ\", \"газ\", \"Газ\"]).interpretation(Mashina.label.const(\"ГАЗ\")),\n",
    "    caseless_pipeline(gaz_types).interpretation(Mashina.model.normalized().custom(gaz_types.get)).optional(),\n",
    "    caseless_pipeline(years).interpretation(Mashina.date.normalized().custom(years.get)).optional()\n",
    ")\n",
    "uaz_types = {\n",
    "    \"31514\":\"31514\",\n",
    "    \"патриот\":\"патриот\",\n",
    "    \"hunter\":\"hunter\",\n",
    "    \"хантер\":\"hunter\"    \n",
    "}\n",
    "\n",
    "UAZ = rule(\n",
    "    morph_pipeline([\"УАЗ\", \"Уаз\", \"уаз\"]).interpretation(Mashina.label.const(\"УАЗ\")),\n",
    "    caseless_pipeline(uaz_types).interpretation(Mashina.model.normalized().custom(uaz_types.get)).optional(),\n",
    "    caseless_pipeline(years).interpretation(Mashina.date.normalized().custom(years.get)).optional()\n",
    ")\n",
    "volga_types = {\n",
    "    \"31029\":\"31029\",\n",
    "    \"21\":\"21\",\n",
    "    \"2410\":\"2410\",\n",
    "    \"3110\":\"3110\"    \n",
    "}\n",
    "\n",
    "Volga = rule(\n",
    "    morph_pipeline([\"Волга\", \"волга\", \"ВОЛГА\", \"Волгу\", \"волгу\", \"ВОЛГУ\"]).interpretation(Mashina.label.const(\"Волга\")),\n",
    "    caseless_pipeline(volga_types).interpretation(Mashina.model.normalized().custom(volga_types.get)).optional(),\n",
    "    caseless_pipeline(years).interpretation(Mashina.date.normalized().custom(years.get)).optional()\n",
    ")\n",
    "niva_types = {\n",
    "    \"шевроле\":\"шевроле\",\n",
    "    \"4х4\":\"4х4\"\n",
    "}\n",
    "Niva = rule(\n",
    "    morph_pipeline([\"Нива\", \"нива\", \"НИВА\", \"Ниву\", \"ниву\", \"НИВУ\"]).interpretation(Mashina.label.const(\"Нива\")),\n",
    "    caseless_pipeline(niva_types).interpretation(Mashina.model.normalized().custom(niva_types.get)).optional(),\n",
    "    caseless_pipeline(years).interpretation(Mashina.date.normalized().custom(years.get)).optional()\n",
    ")\n",
    "lada_types = {\n",
    "    \"калина\":\"калина\",\n",
    "    \"kalina\":\"калина\",\n",
    "    \"приора\":\"приора\",\n",
    "    \"priora\":\"приора\",\n",
    "    \"гранта\":\"гранта\",\n",
    "    \"granta\":\"гранта\"\n",
    "}\n",
    "Lada = rule(\n",
    "    morph_pipeline([\"Lada\", \"LADA\", \"lada\", \"лада\", \"ЛАДА\", \"Лада\"]).interpretation(Mashina.label.const(\"Лада\")),\n",
    "    caseless_pipeline(lada_types).interpretation(Mashina.model.normalized().custom(lada_types.get)).optional(),\n",
    "    caseless_pipeline(years).interpretation(Mashina.date.normalized().custom(years.get)).optional()\n",
    ")\n",
    "gazel_types = {\n",
    "    \"278412\":\"278412\",\n",
    "    \"фермер\":\"фермер\",\n",
    "    \"бизнес\":\"бизнес\",\n",
    "    \"3302\":\"33023\",\n",
    "    \"33023\":\"33023\",\n",
    "    \"некст\":\"некст\",\n",
    "}\n",
    "Gazel = rule(\n",
    "    morph_pipeline([\"Газель\", \"ГАЗЕЛЬ\", \"газель\", \"ГАЗель\"]).interpretation(Mashina.label.const(\"Газель\")),\n",
    "    caseless_pipeline(gazel_types).interpretation(Mashina.model.normalized().custom(gazel_types.get)).optional(),\n",
    "    caseless_pipeline(years).interpretation(Mashina.date.normalized().custom(years.get)).optional()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "MASHINA = or_(VAZ, GAZ, UAZ, Niva, Lada, Volga, Gazel).interpretation(Mashina)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "parser = Parser(MASHINA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 53726/53726 [06:35<00:00, 135.87it/s]\n"
     ]
    }
   ],
   "source": [
    "matches = []\n",
    "\n",
    "for sent in tqdm(car_data):\n",
    "    for match in parser.findall(sent):\n",
    "        matches.append(match.fact)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def with_opts(item):\n",
    "    if item.model is not None and item.date is not None:\n",
    "        return True\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_matches = [i for i in filter(with_opts, matches)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "220"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(full_matches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "no_repeat = list(set(full_matches))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Марка: Нива,\n",
      "модель: 4х4,\n",
      "год: 2006\n",
      "\n",
      "Марка: Лада,\n",
      "модель: приора,\n",
      "год: 2007\n",
      "\n",
      "Марка: Нива,\n",
      "модель: 4х4,\n",
      "год: 2017\n",
      "\n",
      "Марка: Газель,\n",
      "модель: бизнес,\n",
      "год: 2012\n",
      "\n",
      "Марка: Нива,\n",
      "модель: 4х4,\n",
      "год: 1997\n",
      "\n",
      "Марка: Газель,\n",
      "модель: фермер,\n",
      "год: 2004\n",
      "\n",
      "Марка: Лада,\n",
      "модель: приора,\n",
      "год: 2009\n",
      "\n",
      "Марка: Лада,\n",
      "модель: гранта,\n",
      "год: 2015\n",
      "\n",
      "Марка: УАЗ,\n",
      "модель: патриот,\n",
      "год: 2012\n",
      "\n",
      "Марка: Лада,\n",
      "модель: калина,\n",
      "год: 2014\n",
      "\n",
      "Марка: Лада,\n",
      "модель: приора,\n",
      "год: 2015\n",
      "\n",
      "Марка: Нива,\n",
      "модель: шевроле,\n",
      "год: 2012\n",
      "\n",
      "Марка: ГАЗ,\n",
      "модель: 2112,\n",
      "год: 2004\n",
      "\n",
      "Марка: Лада,\n",
      "модель: калина,\n",
      "год: 2005\n",
      "\n",
      "Марка: Нива,\n",
      "модель: 4х4,\n",
      "год: 2014\n",
      "\n",
      "Марка: ГАЗ,\n",
      "модель: 2106,\n",
      "год: 1997\n",
      "\n",
      "Марка: УАЗ,\n",
      "модель: патриот,\n",
      "год: 2013\n",
      "\n",
      "Марка: Нива,\n",
      "модель: шевроле,\n",
      "год: 2006\n",
      "\n",
      "Марка: Нива,\n",
      "модель: шевроле,\n",
      "год: 2013\n",
      "\n",
      "Марка: Лада,\n",
      "модель: калина,\n",
      "год: 2011\n",
      "\n",
      "Марка: ГАЗ,\n",
      "модель: 2109,\n",
      "год: 2000\n",
      "\n",
      "Марка: УАЗ,\n",
      "модель: hunter,\n",
      "год: 2016\n",
      "\n",
      "Марка: УАЗ,\n",
      "модель: 31514,\n",
      "год: 2000\n",
      "\n",
      "Марка: УАЗ,\n",
      "модель: hunter,\n",
      "год: 2006\n",
      "\n",
      "Марка: УАЗ,\n",
      "модель: hunter,\n",
      "год: 2008\n",
      "\n",
      "Марка: Нива,\n",
      "модель: шевроле,\n",
      "год: 2014\n",
      "\n",
      "Марка: Лада,\n",
      "модель: калина,\n",
      "год: 2007\n",
      "\n",
      "Марка: ГАЗ,\n",
      "модель: 21074,\n",
      "год: 2008\n",
      "\n",
      "Марка: Газель,\n",
      "модель: бизнес,\n",
      "год: 2017\n",
      "\n",
      "Марка: Газель,\n",
      "модель: бизнес,\n",
      "год: 2011\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in no_repeat[:30]:\n",
    "    print(f\"Марка: {i.label},\\nмодель: {i.model},\\nгод: {i.date}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}