02_Triple Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1ef34ecf",
   "metadata": {},
   "source": [
    "Ref: \n",
    "- https://nbviewer.org/url/lope.linguistics.ntu.edu.tw/courses/python4nlp/week6-2.Working.with.Lexical.Data.2.ipynb\n",
    "- https://verbs.colorado.edu/verb-index/VerbNet_Guidelines.pdf\n",
    "- https://verbs.colorado.edu/~mpalmer/projects/verbnet.html\n",
    "- https://www.nltk.org/_modules/nltk/corpus/reader/verbnet.html"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d06af256",
   "metadata": {},
   "source": [
    "### Clustering Subjects and Objects\n",
    "We focus on author (first person pronouns I and we) and the dataset.\n",
    "\n",
    "We do the following in subjects and objects:\n",
    "1. identify \"I\" and \"we\" \n",
    "2. detect (probably not full currently) dataset in part or in ful\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "95447b1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_author(text):\n",
    "    '''\n",
    "    input: full sentence (text)\n",
    "    output: bool of find or not\n",
    "    '''\n",
    "    text_lower = text.lower()\n",
    "    words = text_lower.split()\n",
    "    if (\"i\" in words or \"we\" in words):\n",
    "        return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3eaf07d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_author('we')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c7b2088c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_author('three geographic classifications')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b06f994d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# source 1: the detected named entity\n",
    "# source 2: the coreference of the dataset\n",
    "data_keywords = ['data', 'data\\s*(?:set|base)s?', 'corp(us|ora)', 'tree\\s*bank', \n",
    "            '(?:train|test|validation|testing|trainings?)\\s*(?:set)',\n",
    "            'collections?', 'benchmarks?', 'surveys?', 'samples?', 'stud(y|ies)']\n",
    "import re\n",
    "data_pattern= re.compile(r'\\b(' + '|'.join(data_keywords) + r')\\b', flags = re.IGNORECASE)\n",
    "\n",
    "def find_dataset(text,data_name):\n",
    "    '''\n",
    "    input: full sentence (text) and dataset_prediction (data_name)\n",
    "    output: bool of find or not\n",
    "    '''\n",
    "    # use predicted dataset names to find\n",
    "    data_name_list = data_name.split()\n",
    "    words = text.split()\n",
    "    for data_name_token in data_name_list: # anything match counts\n",
    "        if data_name_token in words:\n",
    "            return True\n",
    "    \n",
    "    # use data citation pattern to find\n",
    "    if re.search(data_pattern,text):\n",
    "        return True\n",
    "    \n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "652f9c43",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_dataset(\"ADR data\",\"ARD\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c5841164",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_dataset(\"ADR data\",\"ARD AMEX\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc0c45ea",
   "metadata": {},
   "source": [
    "### Clustering Relations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2ac2827f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e9ec9f0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "### new: https://verbs.colorado.edu/kest1439/\n",
    "# pre-trained golve\n",
    "# https://radimrehurek.com/gensim/models/word2vec.html\n",
    "\n",
    "glove_verb_sense = pd.DataFrame()\n",
    "\n",
    "with open(\"/nfs/turbo/hrg/glove_verb_sense/glove-sense450.vectors.txt\") as f:\n",
    "    lines = f.readlines()\n",
    "    glove_verb_sense[\"lines\"]=lines\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "30a568d8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2844385 entries, 0 to 2844384\n",
      "Data columns (total 1 columns):\n",
      " #   Column  Dtype \n",
      "---  ------  ----- \n",
      " 0   lines   object\n",
      "dtypes: object(1)\n",
      "memory usage: 21.7+ MB\n"
     ]
    }
   ],
   "source": [
    "glove_verb_sense.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ac11837c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lines</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>the -0.696553 -0.199726 0.125435 0.097190 -0.3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>. -0.234015 0.050176 0.169443 0.535785 -0.0603...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>of 0.104824 -0.606487 0.208627 0.197995 -0.081...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>and 0.022772 0.043362 0.474807 0.367868 0.0506...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>in -0.312536 0.040338 0.555955 -0.090425 -0.48...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               lines\n",
       "0  the -0.696553 -0.199726 0.125435 0.097190 -0.3...\n",
       "1  . -0.234015 0.050176 0.169443 0.535785 -0.0603...\n",
       "2  of 0.104824 -0.606487 0.208627 0.197995 -0.081...\n",
       "3  and 0.022772 0.043362 0.474807 0.367868 0.0506...\n",
       "4  in -0.312536 0.040338 0.555955 -0.090425 -0.48..."
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove_verb_sense.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "21083856",
   "metadata": {},
   "outputs": [],
   "source": [
    "eg_vn = glove_verb_sense.lines[0].split(\" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a342b50b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "101"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(eg_vn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ab3784ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_line_to_vec(x):\n",
    "    x_list = x.split(\" \")[1:]\n",
    "    x_list[-1] = x_list[-1].replace(\"\\n\",\"\")\n",
    "    x_list = [float(i) for i in x_list]\n",
    "    return x_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a4804e5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_verb_sense[\"verb\"] = glove_verb_sense.lines.apply(lambda x: x.split(\" \")[0])\n",
    "glove_verb_sense[\"vector\"] = glove_verb_sense.lines.apply(lambda x: convert_line_to_vec(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "cb10f958",
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_verb_sense_dict = dict(zip(glove_verb_sense.verb, glove_verb_sense.vector))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66901da9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# try to apply:\n",
    "# https://adp.uni.edu/documents/bloomverbscognitiveaffectivepsychomotor.pdf\n",
    "# https://courses.washington.edu/pharm439/Bloomstax.htm\n",
    "# https://www.potsdam.edu/sites/default/files/documents/offices/ie/assessment/Action-Verb-List-For-Writing-Student-Outcomes.pdf\n",
    "# https://tips.uark.edu/blooms-taxonomy-verb-chart/\n",
    "# same thing above - Bloom's taxonomy\n",
    "glove_verb_sense_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "4c9eb90a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import dot\n",
    "from numpy.linalg import norm\n",
    "def cos_sim(List1,List2):\n",
    "    result = dot(List1, List2)/(norm(List1)*norm(List2))\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "089f00c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def verb_sense_sim(Word1,Word2,VerbSenseDict=glove_verb_sense_dict):\n",
    "    Vec1 = VerbSenseDict[Word1]\n",
    "    Vec2 = VerbSenseDict[Word2]\n",
    "    return cos_sim(Vec1,Vec2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "9b1aa0c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7358243262368103\n",
      "-0.2063585409985059\n",
      "0.4616696176270366\n"
     ]
    }
   ],
   "source": [
    "# eg - results very bad...\n",
    "print(verb_sense_sim(\"use\",\"have\"))\n",
    "print(verb_sense_sim(\"use\",\"utilize\"))\n",
    "print(verb_sense_sim(\"use\",\"report\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "aa5d55da",
   "metadata": {},
   "outputs": [],
   "source": [
    "### old\n",
    "from nltk.corpus import verbnet as vn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ef0c1ddf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vn.classids('are') # this shows that we need to use AEO first"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "026fba96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['herd-47.5.2',\n",
       " 'knead-26.5',\n",
       " 'obtain-13.5.2',\n",
       " 'other_cos-45.4',\n",
       " 'shake-22.3-2']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vn.classids('collect') # this shows that we need to use lemma"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "928ca633",
   "metadata": {},
   "source": [
    "bipatite setting? One \"relation\" got matched to multiple verb classes. We can then clustering \"relation\" based on the other group of nodes?\n",
    "\n",
    "https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.bipartite.cluster.clustering.html\n",
    "\n",
    "http://yiling.seas.harvard.edu/wp-content/uploads/ISWC03.pdf\n",
    "\n",
    "https://www.osti.gov/servlets/purl/816202\n",
    "\n",
    "https://cdlib.readthedocs.io/en/latest/reference/classes/bi_node_clustering.html\n",
    "\n",
    "Maybe we should have a subset from https://verbs.colorado.edu/verb-index/VerbNet_Guidelines.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "918b70c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f6288c36",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ref: https://verbs.colorado.edu/verb-index/VerbNet_Guidelines.pdf\n",
    "# https://docs.google.com/spreadsheets/d/18kn2z2df-M4ncUmoHPGqbs5nJyGL-k9R0d2slXdT830/edit?usp=sharing\n",
    "verb_class_df = pd.read_csv(\"VerbNet_LF.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b6eb0c8f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101 entries, 0 to 100\n",
      "Data columns (total 3 columns):\n",
      " #   Column        Non-Null Count  Dtype \n",
      "---  ------        --------------  ----- \n",
      " 0   Class Number  101 non-null    int64 \n",
      " 1   Verb Type     101 non-null    object\n",
      " 2   Verb Class    101 non-null    object\n",
      "dtypes: int64(1), object(2)\n",
      "memory usage: 2.5+ KB\n"
     ]
    }
   ],
   "source": [
    "verb_class_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "287d71da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class Number</th>\n",
       "      <th>Verb Type</th>\n",
       "      <th>Verb Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9</td>\n",
       "      <td>Verbs of Putting</td>\n",
       "      <td>put-­9.1 put_spatial-­9.2 funnel-­9.3 put_dire...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>Verbs of Removing</td>\n",
       "      <td>remove-­10.1 banish-­10.2 clear-­10.3 wipe_man...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11</td>\n",
       "      <td>Verbs of Sending and Carrying</td>\n",
       "      <td>send-­11.1 slide-­11.2 bring-­11.3 carry-­11.4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12</td>\n",
       "      <td>Verbs of Exerting Force: Push/Pull Verbs</td>\n",
       "      <td>push-­12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13</td>\n",
       "      <td>Verbs of Change of Possession</td>\n",
       "      <td>give-­13.1 contribute-­13.2 future_having-­13....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Class Number                                 Verb Type  \\\n",
       "0             9                          Verbs of Putting   \n",
       "1            10                         Verbs of Removing   \n",
       "2            11             Verbs of Sending and Carrying   \n",
       "3            12  Verbs of Exerting Force: Push/Pull Verbs   \n",
       "4            13             Verbs of Change of Possession   \n",
       "\n",
       "                                          Verb Class  \n",
       "0  put-­9.1 put_spatial-­9.2 funnel-­9.3 put_dire...  \n",
       "1  remove-­10.1 banish-­10.2 clear-­10.3 wipe_man...  \n",
       "2  send-­11.1 slide-­11.2 bring-­11.3 carry-­11.4...  \n",
       "3                                           push-­12  \n",
       "4  give-­13.1 contribute-­13.2 future_having-­13....  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "verb_class_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "25a62d5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#verb_class_df.astype({'Class Number': 'int'}).dtypes\n",
    "verb_class = verb_class_df.drop(\"Verb Class\",axis=1).set_index(\"Class Number\").to_dict()['Verb Type']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b08a95af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{9: 'Verbs of Putting',\n",
       " 10: 'Verbs of Removing',\n",
       " 11: 'Verbs of Sending and Carrying',\n",
       " 12: 'Verbs of Exerting Force: Push/Pull Verbs',\n",
       " 13: 'Verbs of Change of Possession',\n",
       " 14: 'Learn Verbs',\n",
       " 15: 'Hold and Keep Verbs',\n",
       " 16: 'Verbs of Concealment',\n",
       " 17: 'Verbs of Throwing',\n",
       " 18: 'Verbs of Contact by Impact',\n",
       " 19: 'Poke Verbs',\n",
       " 20: 'Verbs of Contact: Touch Verbs',\n",
       " 21: 'Verbs of Cutting',\n",
       " 22: 'Verbs of Combining and Attaching',\n",
       " 23: 'Verbs of Separating and Disassembling',\n",
       " 24: 'Verbs of Coloring',\n",
       " 25: 'Image Creation Verbs',\n",
       " 26: 'Verbs of Creation and Transformation',\n",
       " 27: 'Engender Verbs',\n",
       " 28: 'Calve Verbs',\n",
       " 29: 'Verbs with Predicative Complements',\n",
       " 30: 'Verbs of Perception',\n",
       " 31: 'Psych-\\xadVerbs (Verbs of Psychological State)',\n",
       " 32: 'Verbs of Desire',\n",
       " 33: 'Judgment Verbs',\n",
       " 34: 'Verbs of Assessment',\n",
       " 35: 'Verbs of Searching',\n",
       " 36: 'Verbs of Social Interaction',\n",
       " 37: 'Verbs of Communication',\n",
       " 38: 'Verbs of Sounds Made by Animals',\n",
       " 39: 'Verbs of Ingesting',\n",
       " 40: 'Verbs Involving the Body',\n",
       " 41: 'Verbs of Grooming and Bodily Care',\n",
       " 42: 'Verbs of Killing',\n",
       " 43: 'Verbs of Emission',\n",
       " 44: 'Destroy Verbs',\n",
       " 45: 'Verbs of Change of State',\n",
       " 46: 'Lodge Verbs',\n",
       " 47: 'Verbs of Existence',\n",
       " 48: 'Verbs of Appearance, Disappearance, and Occurrence',\n",
       " 49: 'Verbs of Body-\\xadInternal Motion',\n",
       " 50: 'Verbs of Assuming a Position',\n",
       " 51: 'Verbs of Motion',\n",
       " 52: 'Avoid Verbs',\n",
       " 53: 'Verbs of Lingering and Rushing',\n",
       " 54: 'Measure Verbs',\n",
       " 55: 'Aspectual Verbs',\n",
       " 56: 'Weekend Verbs',\n",
       " 57: 'Weather Verbs',\n",
       " 58: 'Verbs of Urging and Begging',\n",
       " 59: 'Force Verbs',\n",
       " 60: 'Order Verbs',\n",
       " 61: 'Try Verbs',\n",
       " 62: 'Wish Verbs',\n",
       " 63: 'Enforce Verbs',\n",
       " 64: 'Allow Verbs',\n",
       " 65: 'Admit Verbs',\n",
       " 66: 'Consume Verbs',\n",
       " 67: 'Forbid Verbs',\n",
       " 68: 'Pay Verbs',\n",
       " 69: 'Refrain Verbs',\n",
       " 70: 'Rely Verbs',\n",
       " 71: 'Conspire Verbs',\n",
       " 72: 'Help Verbs',\n",
       " 73: 'Cooperate Verbs',\n",
       " 74: 'Succeed Verbs',\n",
       " 75: 'Neglect Verbs',\n",
       " 76: 'Limit Verbs',\n",
       " 77: 'Approve Verbs',\n",
       " 78: 'Indicate Verbs',\n",
       " 79: 'Dedicate Verbs',\n",
       " 80: 'Free Verbs',\n",
       " 81: 'Suspect Verbs',\n",
       " 82: 'Withdraw Verbs',\n",
       " 83: 'Cope Verbs',\n",
       " 84: 'Discover Verbs',\n",
       " 85: 'Defend Verbs',\n",
       " 86: 'Verbs of Correlating and Relating',\n",
       " 87: 'Verbs of Focusing and Comprehending',\n",
       " 88: 'Verbs of Caring and Empathizing',\n",
       " 89: 'Settle Verbs',\n",
       " 90: 'Exceed Verbs',\n",
       " 91: 'Matter Verbs',\n",
       " 92: 'Confine Verbs',\n",
       " 93: 'Adopt Verbs',\n",
       " 94: 'Risk Verbs',\n",
       " 95: 'Acquiesce Verbs',\n",
       " 96: 'Addict Verbs',\n",
       " 97: 'Verbs of Basing and Deducing',\n",
       " 98: 'Confront Verbs',\n",
       " 99: 'Ensure Verbs',\n",
       " 100: 'Own Verbs',\n",
       " 101: 'Patent Verbs',\n",
       " 102: 'Promote Verbs',\n",
       " 103: 'Require Verbs',\n",
       " 104: 'Verbs of Spending Time',\n",
       " 105: 'Use Verbs',\n",
       " 106: 'Void Verbs',\n",
       " 107: 'Involve Verbs',\n",
       " 108: 'Multiply Verbs',\n",
       " 109: 'Seem Verbs'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "verb_class # we may want to merge! - according to the context of data reference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5a322dc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_classes(classids):\n",
    "    \"\"\"\n",
    "    input VerbNet classids (long)\n",
    "    output a set of VerbNet classes (parenent-level, short)\n",
    "    \"\"\"\n",
    "    classes = set()\n",
    "    for classid in classids:\n",
    "        # remove the word itself\n",
    "        this_classid_long = classid.split(\"-\")[1]\n",
    "        # get the class -- the string upto the first non-digit\n",
    "        this_classid_short = int(re.search(r'(\\d+)',this_classid_long).group(1))\n",
    "        classes.add(this_classid_short)\n",
    "    return classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "386a36fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['force-59',\n",
       " 'force-59-1',\n",
       " 'performance-26.7-2-1',\n",
       " 'remove-10.1',\n",
       " 'scribble-25.2',\n",
       " 'split-23.2']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# example\n",
    "vn.classids('draw')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2acf1a2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{10, 23, 25, 26, 59}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_classes(vn.classids('draw'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c84bdd9",
   "metadata": {},
   "source": [
    "### Convert from AEO\n",
    "ref: https://github.com/lizhouf/semantic_triplets/blob/main/scr/add_aeo.py\n",
    "\n",
    "clauses are characterized by:\n",
    "- temporal organization (the order in which the subject narrates events and actions in the story), \n",
    "- evaluative description (personal assessments made by the narrator), and \n",
    "- contextual orientation (usually information provided by the narrator that helps orient the listener)\n",
    "\n",
    "ref: Labov and Waletsky 1997 Labov, William, and Joshua Waletzky. 1997. “Narrative Analysis: Oral Versions of Personal Experience.” Journal of Narrative & Life History 7 (1–4): 3–38.\n",
    "\n",
    "We have: \n",
    "- Active Agency\n",
    "- Passive Agency\n",
    "- Possible Agency\n",
    "- Evaluative Description\n",
    "- Contextual Orientation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d82189e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "from spacy.matcher import Matcher\n",
    "from spacy.symbols import dobj, obj, pobj, acomp, ccomp, pcomp, xcomp, conj, acomp, ccomp, pcomp, xcomp, advmod, amod\n",
    "from spacy.symbols import neg, det, aux, prep, poss, nsubj, nsubjpass, csubj, csubjpass, det, prt\n",
    "from spacy.symbols import VERB, AUX, DET, ADP, ADV, ADJ, NOUN, PRON, PROPN, PART\n",
    "from spacy.tokenizer import Tokenizer\n",
    "from spacy.lang.char_classes import ALPHA, ALPHA_LOWER, ALPHA_UPPER, CONCAT_QUOTES, LIST_ELLIPSES, LIST_ICONS\n",
    "from spacy.util import compile_infix_regex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d4294f8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Example Key Words\n",
    "'''\n",
    "evaluation_verbs = [\"feel\",\"smell\",\"taste\",\"look\",\"hear\",\"see\",\"think\",\"know\"]\n",
    "orientation_verbs = [\"remember\",\"bear\",\"grow\",\"belong\"]\n",
    "imagine_verbs = [\"want\",\"should\",\"would\",\"could\",\"can\",\"might\",\"may\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8edf64fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_cat(this_rel, this_obj):\n",
    "    '''\n",
    "    input spaCy Spans this_rel, this_obj \n",
    "    output category result\n",
    "    '''\n",
    "    \n",
    "    # initate category result\n",
    "    \n",
    "    this_cat = \"\"\n",
    "\n",
    "    # initiate the rule components\n",
    "\n",
    "    rel_has_evaluation = 0\n",
    "    rel_has_orientation = 0\n",
    "    rel_has_imagine = 0\n",
    "\n",
    "    rel_has_be = 0\n",
    "    rel_has_have = 0\n",
    "    rel_has_to = 0\n",
    "\n",
    "    rel_has_neg = 0\n",
    "\n",
    "    rel_has_VBG = 0\n",
    "    rel_num_verb = 0\n",
    "\n",
    "    obj_is_adj = 0  # only adj, no NOUN+\n",
    "\n",
    "    obj_has_no = 0\n",
    "    \n",
    "    # give value\n",
    "    for rel in this_rel:\n",
    "\n",
    "        # rel lemmas\n",
    "        try:\n",
    "            if rel.lemma_ in evaluation_verbs:\n",
    "                rel_has_evaluation = 1\n",
    "            if rel.lemma_ in imagine_verbs:\n",
    "                rel_has_imagine = 1\n",
    "            if rel.lemma_ in orientation_verbs:\n",
    "                rel_has_orientation = 1\n",
    "            if rel.lemma_ == \"be\":\n",
    "                rel_has_be = 1\n",
    "            if rel.lemma_ == \"have\":\n",
    "                rel_has_have = 1\n",
    "            if rel.lemma_ == \"to\":\n",
    "                rel_has_to = 1\n",
    "        except:  # avoid no lemma\n",
    "            0\n",
    "\n",
    "        # rel dep\n",
    "        try:\n",
    "            if rel.dep == neg:\n",
    "                rel_has_neg = 1\n",
    "        except:\n",
    "            0\n",
    "\n",
    "        # rel pos\n",
    "        try:\n",
    "            if (rel.pos == VERB or rel.pos == AUX):\n",
    "                rel_num_verb = rel_num_verb + 1\n",
    "        except:\n",
    "            0\n",
    "\n",
    "        # rel tag\n",
    "        try:\n",
    "            if rel.tag_ == \"VBG\":\n",
    "                rel_has_VBG = 1\n",
    "        except:\n",
    "            0\n",
    "\n",
    "    for obj in this_obj:\n",
    "        \n",
    "        # obj lemma\n",
    "        try:\n",
    "            if obj.lemma_ == \"no\":\n",
    "                obj_has_no = 1\n",
    "        except:\n",
    "            0\n",
    "\n",
    "    for obj in this_obj:  # seperate, want to break\n",
    "        # obj pos\n",
    "        try:\n",
    "            if obj.pos == ADJ:\n",
    "                obj_is_adj = 1\n",
    "            if obj.pos in [NOUN,PRON,PROPN]:\n",
    "                obj_is_adj = 0\n",
    "                break\n",
    "        except:\n",
    "            0\n",
    "\n",
    "    # judge:\n",
    "\n",
    "    # fixed words\n",
    "    if rel_has_evaluation and obj_is_adj:\n",
    "        this_cat =\"Evaluation\"\n",
    "    elif rel_has_imagine:\n",
    "        this_cat =\"Agency_Possible\"\n",
    "    elif rel_has_orientation:\n",
    "        this_cat =\"Orientation\"\n",
    "\n",
    "    # neg\n",
    "    elif rel_has_neg or obj_has_no:\n",
    "        this_cat =\"Orientation\"\n",
    "\n",
    "    # have\n",
    "    elif rel_has_have:\n",
    "        if rel_has_to:\n",
    "            this_cat =\"Agency_Passive\" # no longer coercive\n",
    "        else:\n",
    "            this_cat =\"Orientation\"\n",
    "\n",
    "    # be\n",
    "    elif rel_has_be:\n",
    "        if obj_is_adj:\n",
    "            this_cat =\"Evaluation\"\n",
    "        elif rel_has_VBG:\n",
    "            this_cat =\"Agency_Active\"\n",
    "        elif rel_num_verb > 1:\n",
    "            this_cat =\"Agency_Passive\"\n",
    "        elif rel_num_verb == 1:\n",
    "            this_cat =\"Orientation\"\n",
    "\n",
    "    # if none of the above, then assign active:\n",
    "    else:\n",
    "        this_cat = \"Agency_Active\"\n",
    "\n",
    "    return this_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "82c3f8e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Agency_Passive'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# subject is dataset\n",
    "eg_rel1 = nlp('are drawn from')\n",
    "eg_obj1 = nlp('National Organizations Survey')\n",
    "get_cat(eg_rel1,eg_obj1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "8d5ca0d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "obtain VERB\n",
      "name NOUN\n",
      "for ADP\n"
     ]
    }
   ],
   "source": [
    "for rel in nlp('obtain name For'):\n",
    "    print(rel.lemma_,rel.pos_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4fa6e1f4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Agency_Active'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# subject is we\n",
    "eg_rel2 = nlp('obtain name For')\n",
    "eg_obj2 = nlp('non-ADR shares')\n",
    "get_cat(eg_rel2,eg_obj2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "cc463ebc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Agency_Passive'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# subject is dataset\n",
    "eg_rel3 = nlp('are collected from')\n",
    "eg_obj3 = nlp('major depositary bank websites')\n",
    "get_cat(eg_rel3,eg_obj3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "34089b98",
   "metadata": {},
   "outputs": [],
   "source": [
    "# to do: find more examples"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}