examples/Beaver/tensor-Authors-Words-Time.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4d57e6e4-657d-45e0-9d8e-ff050047f3e0",
   "metadata": {},
   "source": [
    "# Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7e3ccf65-3f76-4acd-a516-c4f6a9e16982",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 940 entries, 0 to 939\n",
      "Data columns (total 9 columns):\n",
      " #   Column          Non-Null Count  Dtype \n",
      "---  ------          --------------  ----- \n",
      " 0   eid             940 non-null    object\n",
      " 1   title           940 non-null    object\n",
      " 2   countries       940 non-null    object\n",
      " 3   year            940 non-null    int64 \n",
      " 4   abstract        940 non-null    object\n",
      " 5   authors         940 non-null    object\n",
      " 6   author_ids      940 non-null    object\n",
      " 7   references      843 non-null    object\n",
      " 8   clean_abstract  940 non-null    object\n",
      "dtypes: int64(1), object(8)\n",
      "memory usage: 66.2+ KB\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os \n",
    "df = pd.read_csv(os.path.join(\"..\", \"..\", \"data\", \"sample.csv\"))\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "144efd0d-ce13-4c71-90ad-4efb01e375ae",
   "metadata": {},
   "source": [
    "# Build Tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4db0d643-83a1-48d7-a819-452d04251789",
   "metadata": {},
   "outputs": [],
   "source": [
    "from TELF.pre_processing import Beaver\n",
    "\n",
    "beaver = Beaver()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "797965f7-5e96-48ed-9fe2-1aaf46333513",
   "metadata": {},
   "source": [
    "First let's get our vocabulary from the documents words matrix. **This step is optional!**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b60c315b-8ca1-42c5-9831-40293aab6657",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 104 ms, sys: 121 ms, total: 225 ms\n",
      "Wall time: 16.7 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "466"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "settings = {\n",
    "    \"dataset\":df,\n",
    "    \"target_column\":\"clean_abstract\",\n",
    "    \"min_df\":5,\n",
    "    \"max_df\":0.5,\n",
    "}\n",
    "\n",
    "%time vocabulary = beaver.get_vocabulary(**settings)\n",
    "len(vocabulary)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7639a30f-cfc4-4528-87df-f6a3a1eb682d",
   "metadata": {},
   "source": [
    "Now we can use this vocabulary when building the tensor:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f95cf8c1-7226-4af2-877f-5f24503b9c3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "settings = {\n",
    "    \"dataset\":df,\n",
    "    \"vocabulary\":vocabulary,\n",
    "    \"target_columns\":(\"author_ids\", \"clean_abstract\", \"year\"),\n",
    "    \"tfidf_transformer\":True,\n",
    "    \"unfold_at\":1,\n",
    "    \"save_path\":None,\n",
    "    \"verbose\":False\n",
    "}\n",
    "\n",
    "X, author_ids, vocabulary, time = beaver.something_words_time(**settings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0d9621e9-2d29-4f70-b228-f15dcdb11c25",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7054, 466, 29)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f3a46bb2-75b0-4b30-b428-a0551e81ac4d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7054"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(author_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6abffd56-c693-46ef-845d-f5a71c707635",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "466"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocabulary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1c475de8-ff7b-4234-9519-e17bda38e225",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "bcffbb01-c3d4-42cb-85f0-e8fb692ccc8f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['128pb',\n",
       " '32x',\n",
       " 'ability',\n",
       " 'abstain',\n",
       " 'accelerate',\n",
       " 'accuracy',\n",
       " 'accurate',\n",
       " 'acquisition',\n",
       " 'activity',\n",
       " 'addition']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocabulary[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d1b63390-3ef3-412e-9d7b-f32859d506d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['3df61b32-6781-11ee-b983-4ab2673ea3f0',\n",
       " '3df61c18-6781-11ee-b983-4ab2673ea3f0',\n",
       " '3df61c4a-6781-11ee-b983-4ab2673ea3f0',\n",
       " '3df61c72-6781-11ee-b983-4ab2673ea3f0',\n",
       " '3df61c9a-6781-11ee-b983-4ab2673ea3f0',\n",
       " '3df61cc2-6781-11ee-b983-4ab2673ea3f0',\n",
       " '3df61cea-6781-11ee-b983-4ab2673ea3f0',\n",
       " '3df61d1c-6781-11ee-b983-4ab2673ea3f0',\n",
       " '3df61d3a-6781-11ee-b983-4ab2673ea3f0',\n",
       " '3df61d62-6781-11ee-b983-4ab2673ea3f0']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "author_ids[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2245a219-ba13-4dee-823c-b95bb9c9eb71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "time[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32cfd330-ec59-4772-8590-0d67260bf60c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "TELF",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}