052_Sentiment_Analysis_Transf.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "eed9c1b1-a2f7-469f-8fc0-b377ab2f6f19",
   "metadata": {},
   "source": [
    "# **NB 5.2: Sentiment Analysis using Transformer**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "090f6446-d7a3-4150-ab3e-280b8bb560d9",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Part 0: Importing Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8035ca05-4b3b-4193-9f6e-ff11b9eaaf8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer, HashingVectorizer, TfidfTransformer, TfidfVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
    "from sklearn.linear_model import LogisticRegression, SGDClassifier\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn import metrics\n",
    "\n",
    "from joblib import dump, load\n",
    "from joblib import Parallel, delayed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6429bcb8-3b40-4846-8516-d22653f99f3a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Available CPUs: 16\n",
      "INFO: Pandarallel will run on 15 workers.\n",
      "INFO: Pandarallel will use standard multiprocessing data transfer (pipe) to transfer data between the main process and workers.\n"
     ]
    }
   ],
   "source": [
    "#pip install pandarallel\n",
    "import multiprocessing\n",
    "\n",
    "num_processors = multiprocessing.cpu_count()\n",
    "print(f'Available CPUs: {num_processors}')\n",
    "\n",
    "import pandarallel\n",
    "from pandarallel import pandarallel\n",
    "pandarallel.initialize(nb_workers=num_processors-1, use_memory_fs=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6af7cf21-b64a-480f-9a48-212c45a13c8a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bucket nlp_final_project_kshitijm connected.\n"
     ]
    }
   ],
   "source": [
    "# Imports the Google Cloud client library\n",
    "from google.cloud import storage\n",
    "# Instantiates a client\n",
    "storage_client = storage.Client()\n",
    "\n",
    "# The name for the new bucket\n",
    "bucket_name = \"nlp_final_project_kshitijm\"\n",
    "\n",
    "# Creates the new bucket\n",
    "bucket = storage_client.bucket(bucket_name)\n",
    "print(f\"Bucket {bucket.name} connected.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "59dab5ad-b745-4a50-a5e6-ceb28e10340d",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Part 1: Importing transformer from hugging face hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a7597abb-b2ca-43be-99e6-00231dfb4687",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "56351b28-c4c2-4e1a-874e-b34dd26449f7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-21 17:17:46.405635: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-05-21 17:18:00.244809: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/nccl2/lib:/usr/local/cuda/extras/CUPTI/lib64\n",
      "2023-05-21 17:18:00.246291: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/nccl2/lib:/usr/local/cuda/extras/CUPTI/lib64\n",
      "2023-05-21 17:18:00.246314: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"
     ]
    }
   ],
   "source": [
    "# pip install transformers\n",
    "from transformers import pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8f2aea2e-2bc7-43f6-9f33-6faba8ead1de",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"mrm8488/distilroberta-finetuned-financial-news-sentiment-analysis\")\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\"mrm8488/distilroberta-finetuned-financial-news-sentiment-analysis\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ef6b13b4-06fb-4fb2-b449-f8502927e12f",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = [\"Chatgpt has made my life really good\",\"Openai stock stock is going down\"]\n",
    "token=tokenizer(prompt, padding='max_length', max_length=256, truncation=True, return_tensors='pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "37863cef-95f5-4c54-9cbd-ec888d4d8692",
   "metadata": {},
   "outputs": [],
   "source": [
    "output=model(**token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c90a463d-7a25-40c0-9928-40f814a605bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-2.4181, -2.8373,  5.7378],\n",
       "        [ 4.3566, -1.3070, -3.4122]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output.logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a75b3655-597c-41d8-acda-249a5a2afb43",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[2.8689814e-04 1.8867313e-04 9.9952447e-01]\n",
      " [9.9612242e-01 3.4564650e-03 4.2109768e-04]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:2: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "import torch.nn.functional as F\n",
    "probs=F.softmax(output.logits)\n",
    "probs=probs.detach().numpy()\n",
    "print(probs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46465c6d-ee8c-4e57-b992-361ac7358d56",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Part 2: Predicting sentiments for our dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "3f433085-bec7-4116-b7b9-6dfed4775fea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>date</th>\n",
       "      <th>language</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>cleaned_text</th>\n",
       "      <th>article_source</th>\n",
       "      <th>clean_title</th>\n",
       "      <th>title_tokens</th>\n",
       "      <th>cleaned_text_tokens</th>\n",
       "      <th>year_pub</th>\n",
       "      <th>month</th>\n",
       "      <th>month-year</th>\n",
       "      <th>flag_relevant</th>\n",
       "      <th>num_tokens</th>\n",
       "      <th>rake_phrases_articles</th>\n",
       "      <th>rake_phrases_joined</th>\n",
       "      <th>final_topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://fusionscienceacademy.com/artificial-in...</td>\n",
       "      <td>2020-01-30</td>\n",
       "      <td>en</td>\n",
       "      <td>Artificial Intelligence (AI) in Social Media  ...</td>\n",
       "      <td>\\n\\nArtificial Intelligence (AI) in Social Med...</td>\n",
       "      <td>Artificial Intelligence AI in Social MediaMar...</td>\n",
       "      <td>2026 – Fusion Science Academy</td>\n",
       "      <td>Artificial Intelligence (AI) in Social Media  ...</td>\n",
       "      <td>['artificial', 'intelligence', 'ai', 'social',...</td>\n",
       "      <td>['artificial', 'intelligence', 'ai', 'social',...</td>\n",
       "      <td>2020</td>\n",
       "      <td>1</td>\n",
       "      <td>Jan 2020</td>\n",
       "      <td>1</td>\n",
       "      <td>22283</td>\n",
       "      <td>[burkert fluid control systems emerson electri...</td>\n",
       "      <td>burkert fluid control systems emerson electric...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://health.economictimes.indiatimes.com/ne...</td>\n",
       "      <td>2020-01-10</td>\n",
       "      <td>en</td>\n",
       "      <td>artificial intelligence: Researchers develop A...</td>\n",
       "      <td>\\n\\nartificial intelligence: Researchers devel...</td>\n",
       "      <td>artificial intelligence: Researchers develop ...</td>\n",
       "      <td>None</td>\n",
       "      <td>artificial intelligence: Researchers develop A...</td>\n",
       "      <td>['artificial', 'intelligence', 'researchers', ...</td>\n",
       "      <td>['artificial', 'intelligence', 'researchers', ...</td>\n",
       "      <td>2020</td>\n",
       "      <td>1</td>\n",
       "      <td>Jan 2020</td>\n",
       "      <td>1</td>\n",
       "      <td>8087</td>\n",
       "      <td>[economic times ethealthworldhome news hospita...</td>\n",
       "      <td>economic times ethealthworldhome news hospital...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://heraldpublicist.com/bet-gil-on-ai-fina...</td>\n",
       "      <td>2020-01-15</td>\n",
       "      <td>en</td>\n",
       "      <td>Bet Gil on AI Final Fantasy Tactics Matches in...</td>\n",
       "      <td>\\n\\nBet Gil on AI Final Fantasy Tactics Matche...</td>\n",
       "      <td>Bet Gil on AI Final Fantasy Tactics Matches i...</td>\n",
       "      <td>Herald Publicist</td>\n",
       "      <td>Bet Gil on AI Final Fantasy Tactics Matches in...</td>\n",
       "      <td>['bet', 'gil', 'ai', 'final', 'fantasy', 'tact...</td>\n",
       "      <td>['bet', 'gil', 'ai', 'final', 'fantasy', 'tact...</td>\n",
       "      <td>2020</td>\n",
       "      <td>1</td>\n",
       "      <td>Jan 2020</td>\n",
       "      <td>1</td>\n",
       "      <td>4458</td>\n",
       "      <td>[hilarious twitch streamnewstechnologycricketp...</td>\n",
       "      <td>hilarious twitch streamnewstechnologycricketpo...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://honestversion.com/2020/01/24/growth-of...</td>\n",
       "      <td>2020-01-24</td>\n",
       "      <td>en</td>\n",
       "      <td>Growth of Cloud Telecommunication AI market in...</td>\n",
       "      <td>\\n\\nGrowth of Cloud Telecommunication AI marke...</td>\n",
       "      <td>Growth of Cloud  AI market in global industry...</td>\n",
       "      <td>Honest Version</td>\n",
       "      <td>Growth of Cloud Telecommunication AI market in...</td>\n",
       "      <td>['growth', 'cloud', 'ai', 'market', 'global', ...</td>\n",
       "      <td>['growth', 'cloud', 'ai', 'market', 'global', ...</td>\n",
       "      <td>2020</td>\n",
       "      <td>1</td>\n",
       "      <td>Jan 2020</td>\n",
       "      <td>1</td>\n",
       "      <td>6048</td>\n",
       "      <td>[cloud telecommunication ai market strategic a...</td>\n",
       "      <td>cloud telecommunication ai market strategic as...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://marketresearchsheets.com/2020/01/31/gl...</td>\n",
       "      <td>2020-01-31</td>\n",
       "      <td>en</td>\n",
       "      <td>Global Artificial Intelligence as a Service Ma...</td>\n",
       "      <td>\\n\\nGlobal Artificial Intelligence as a Servic...</td>\n",
       "      <td>Global Artificial Intelligence as a Service M...</td>\n",
       "      <td>Market Research Sheets</td>\n",
       "      <td>Global Artificial Intelligence as a Service Ma...</td>\n",
       "      <td>['global', 'artificial', 'intelligence', 'serv...</td>\n",
       "      <td>['global', 'artificial', 'intelligence', 'serv...</td>\n",
       "      <td>2020</td>\n",
       "      <td>1</td>\n",
       "      <td>Jan 2020</td>\n",
       "      <td>1</td>\n",
       "      <td>10098</td>\n",
       "      <td>[service market trendsyou may also like news g...</td>\n",
       "      <td>service market trendsyou may also like news gl...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 url       date language  \\\n",
       "0  https://fusionscienceacademy.com/artificial-in... 2020-01-30       en   \n",
       "1  https://health.economictimes.indiatimes.com/ne... 2020-01-10       en   \n",
       "2  https://heraldpublicist.com/bet-gil-on-ai-fina... 2020-01-15       en   \n",
       "3  https://honestversion.com/2020/01/24/growth-of... 2020-01-24       en   \n",
       "4  https://marketresearchsheets.com/2020/01/31/gl... 2020-01-31       en   \n",
       "\n",
       "                                               title  \\\n",
       "0  Artificial Intelligence (AI) in Social Media  ...   \n",
       "1  artificial intelligence: Researchers develop A...   \n",
       "2  Bet Gil on AI Final Fantasy Tactics Matches in...   \n",
       "3  Growth of Cloud Telecommunication AI market in...   \n",
       "4  Global Artificial Intelligence as a Service Ma...   \n",
       "\n",
       "                                                text  \\\n",
       "0  \\n\\nArtificial Intelligence (AI) in Social Med...   \n",
       "1  \\n\\nartificial intelligence: Researchers devel...   \n",
       "2  \\n\\nBet Gil on AI Final Fantasy Tactics Matche...   \n",
       "3  \\n\\nGrowth of Cloud Telecommunication AI marke...   \n",
       "4  \\n\\nGlobal Artificial Intelligence as a Servic...   \n",
       "\n",
       "                                        cleaned_text  \\\n",
       "0   Artificial Intelligence AI in Social MediaMar...   \n",
       "1   artificial intelligence: Researchers develop ...   \n",
       "2   Bet Gil on AI Final Fantasy Tactics Matches i...   \n",
       "3   Growth of Cloud  AI market in global industry...   \n",
       "4   Global Artificial Intelligence as a Service M...   \n",
       "\n",
       "                  article_source  \\\n",
       "0  2026 – Fusion Science Academy   \n",
       "1                           None   \n",
       "2               Herald Publicist   \n",
       "3                 Honest Version   \n",
       "4         Market Research Sheets   \n",
       "\n",
       "                                         clean_title  \\\n",
       "0  Artificial Intelligence (AI) in Social Media  ...   \n",
       "1  artificial intelligence: Researchers develop A...   \n",
       "2  Bet Gil on AI Final Fantasy Tactics Matches in...   \n",
       "3  Growth of Cloud Telecommunication AI market in...   \n",
       "4  Global Artificial Intelligence as a Service Ma...   \n",
       "\n",
       "                                        title_tokens  \\\n",
       "0  ['artificial', 'intelligence', 'ai', 'social',...   \n",
       "1  ['artificial', 'intelligence', 'researchers', ...   \n",
       "2  ['bet', 'gil', 'ai', 'final', 'fantasy', 'tact...   \n",
       "3  ['growth', 'cloud', 'ai', 'market', 'global', ...   \n",
       "4  ['global', 'artificial', 'intelligence', 'serv...   \n",
       "\n",
       "                                 cleaned_text_tokens  year_pub  month  \\\n",
       "0  ['artificial', 'intelligence', 'ai', 'social',...      2020      1   \n",
       "1  ['artificial', 'intelligence', 'researchers', ...      2020      1   \n",
       "2  ['bet', 'gil', 'ai', 'final', 'fantasy', 'tact...      2020      1   \n",
       "3  ['growth', 'cloud', 'ai', 'market', 'global', ...      2020      1   \n",
       "4  ['global', 'artificial', 'intelligence', 'serv...      2020      1   \n",
       "\n",
       "  month-year  flag_relevant  num_tokens  \\\n",
       "0   Jan 2020              1       22283   \n",
       "1   Jan 2020              1        8087   \n",
       "2   Jan 2020              1        4458   \n",
       "3   Jan 2020              1        6048   \n",
       "4   Jan 2020              1       10098   \n",
       "\n",
       "                               rake_phrases_articles  \\\n",
       "0  [burkert fluid control systems emerson electri...   \n",
       "1  [economic times ethealthworldhome news hospita...   \n",
       "2  [hilarious twitch streamnewstechnologycricketp...   \n",
       "3  [cloud telecommunication ai market strategic a...   \n",
       "4  [service market trendsyou may also like news g...   \n",
       "\n",
       "                                 rake_phrases_joined  final_topic  \n",
       "0  burkert fluid control systems emerson electric...            6  \n",
       "1  economic times ethealthworldhome news hospital...            1  \n",
       "2  hilarious twitch streamnewstechnologycricketpo...            1  \n",
       "3  cloud telecommunication ai market strategic as...            6  \n",
       "4  service market trendsyou may also like news gl...            6  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_filt_2020=pd.read_parquet('gs://nlp_final_project_kshitijm/00_Data/NLP_FP_Data5_2020_Topics.parquet')\n",
    "df_filt_2020.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "31d1928c-eccb-4dc5-87cf-7a51de93c755",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "def classify_sentiment(text):\n",
    "    # Split or truncate the text if it exceeds the maximum sequence length\n",
    "    # max_seq_length = 512\n",
    "    token=tokenizer(text, padding='max_length', max_length=512, truncation=True, return_tensors='pt')\n",
    "    output=model(**token)\n",
    "    probs=F.softmax(output.logits)\n",
    "    probs=probs.detach().numpy()\n",
    "    return probs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "ede029a8-e854-4b85-bf84-31d0b6384221",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/100 [00:00<?, ?it/s]/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:8: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  \n",
      "100%|██████████| 100/100 [00:15<00:00,  6.56it/s]\n"
     ]
    }
   ],
   "source": [
    "# Apply the classification to the dataframe with progress tracker\n",
    "tqdm.pandas()\n",
    "df_filt_2020_samp=df_filt_2020_samp.sample(100)\n",
    "df_filt_2020_samp['sentiment_text'] = df_filt_2020_samp['cleaned_text'].progress_apply(classify_sentiment)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "af1b4dc6-cfa0-4fd1-ab00-bfc7ab0cae74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    71\n",
       "2    24\n",
       "0     5\n",
       "Name: sentiment_label, dtype: int64"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_filt_2020_samp['sentiment_label']=df_filt_2020_samp['sentiment_text'].apply(lambda x: np.argmax(x[0]))\n",
    "df_filt_2020_samp['sentiment_label'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4c5421d-3f06-48e6-8132-8e5f4385e489",
   "metadata": {},
   "source": [
    "### Running on 2020 dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "f7b37077-e511-442b-a0de-3de6d57df82d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/32667 [00:00<?, ?it/s]/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:8: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  \n",
      "100%|██████████| 32667/32667 [1:22:46<00:00,  6.58it/s]\n",
      "100%|██████████| 32667/32667 [00:00<00:00, 294287.02it/s]\n"
     ]
    }
   ],
   "source": [
    "df_filt_2020['sent_probs']=df_filt_2020['cleaned_text'].progress_apply(classify_sentiment)\n",
    "df_filt_2020['sent_label']=df_filt_2020['sent_probs'].progress_apply(lambda x: np.argmax(x[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "d6e45323-d3ea-4657-be46-7da772c3444c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    22548\n",
       "2     9429\n",
       "0      690\n",
       "Name: sent_label, dtype: int64"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_filt_2020['sent_label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "89bb012d-6d4d-4554-bc1e-6971816efa92",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 57.5 s, sys: 392 ms, total: 57.9 s\n",
      "Wall time: 1min 22s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_filt_2020.to_csv('gs://nlp_final_project_kshitijm/00_Data/NLP_FP_Data5_2020_Topics_Sentiments.csv')                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7cd4e749-9a4a-4dcf-b073-a0ce5a920949",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>url</th>\n",
       "      <th>date</th>\n",
       "      <th>language</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>cleaned_text</th>\n",
       "      <th>article_source</th>\n",
       "      <th>clean_title</th>\n",
       "      <th>title_tokens</th>\n",
       "      <th>...</th>\n",
       "      <th>year_pub</th>\n",
       "      <th>month</th>\n",
       "      <th>month-year</th>\n",
       "      <th>flag_relevant</th>\n",
       "      <th>num_tokens</th>\n",
       "      <th>rake_phrases_articles</th>\n",
       "      <th>rake_phrases_joined</th>\n",
       "      <th>final_topic</th>\n",
       "      <th>sent_probs</th>\n",
       "      <th>sent_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>https://fusionscienceacademy.com/artificial-in...</td>\n",
       "      <td>2020-01-30</td>\n",
       "      <td>en</td>\n",
       "      <td>Artificial Intelligence (AI) in Social Media  ...</td>\n",
       "      <td>\\n\\nArtificial Intelligence (AI) in Social Med...</td>\n",
       "      <td>Artificial Intelligence AI in Social MediaMar...</td>\n",
       "      <td>2026 – Fusion Science Academy</td>\n",
       "      <td>Artificial Intelligence (AI) in Social Media  ...</td>\n",
       "      <td>['artificial', 'intelligence', 'ai', 'social',...</td>\n",
       "      <td>...</td>\n",
       "      <td>2020</td>\n",
       "      <td>1</td>\n",
       "      <td>Jan 2020</td>\n",
       "      <td>1</td>\n",
       "      <td>22283</td>\n",
       "      <td>['burkert fluid control systems emerson electr...</td>\n",
       "      <td>burkert fluid control systems emerson electric...</td>\n",
       "      <td>6</td>\n",
       "      <td>[[2.1574432e-04 9.9863547e-01 1.1488239e-03]]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>https://health.economictimes.indiatimes.com/ne...</td>\n",
       "      <td>2020-01-10</td>\n",
       "      <td>en</td>\n",
       "      <td>artificial intelligence: Researchers develop A...</td>\n",
       "      <td>\\n\\nartificial intelligence: Researchers devel...</td>\n",
       "      <td>artificial intelligence: Researchers develop ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>artificial intelligence: Researchers develop A...</td>\n",
       "      <td>['artificial', 'intelligence', 'researchers', ...</td>\n",
       "      <td>...</td>\n",
       "      <td>2020</td>\n",
       "      <td>1</td>\n",
       "      <td>Jan 2020</td>\n",
       "      <td>1</td>\n",
       "      <td>8087</td>\n",
       "      <td>['economic times ethealthworldhome news hospit...</td>\n",
       "      <td>economic times ethealthworldhome news hospital...</td>\n",
       "      <td>1</td>\n",
       "      <td>[[1.9508268e-04 8.0524624e-04 9.9899966e-01]]</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                                url        date  \\\n",
       "0           0  https://fusionscienceacademy.com/artificial-in...  2020-01-30   \n",
       "1           1  https://health.economictimes.indiatimes.com/ne...  2020-01-10   \n",
       "\n",
       "  language                                              title  \\\n",
       "0       en  Artificial Intelligence (AI) in Social Media  ...   \n",
       "1       en  artificial intelligence: Researchers develop A...   \n",
       "\n",
       "                                                text  \\\n",
       "0  \\n\\nArtificial Intelligence (AI) in Social Med...   \n",
       "1  \\n\\nartificial intelligence: Researchers devel...   \n",
       "\n",
       "                                        cleaned_text  \\\n",
       "0   Artificial Intelligence AI in Social MediaMar...   \n",
       "1   artificial intelligence: Researchers develop ...   \n",
       "\n",
       "                  article_source  \\\n",
       "0  2026 – Fusion Science Academy   \n",
       "1                            NaN   \n",
       "\n",
       "                                         clean_title  \\\n",
       "0  Artificial Intelligence (AI) in Social Media  ...   \n",
       "1  artificial intelligence: Researchers develop A...   \n",
       "\n",
       "                                        title_tokens  ... year_pub  month  \\\n",
       "0  ['artificial', 'intelligence', 'ai', 'social',...  ...     2020      1   \n",
       "1  ['artificial', 'intelligence', 'researchers', ...  ...     2020      1   \n",
       "\n",
       "   month-year flag_relevant  num_tokens  \\\n",
       "0    Jan 2020             1       22283   \n",
       "1    Jan 2020             1        8087   \n",
       "\n",
       "                               rake_phrases_articles  \\\n",
       "0  ['burkert fluid control systems emerson electr...   \n",
       "1  ['economic times ethealthworldhome news hospit...   \n",
       "\n",
       "                                 rake_phrases_joined final_topic  \\\n",
       "0  burkert fluid control systems emerson electric...           6   \n",
       "1  economic times ethealthworldhome news hospital...           1   \n",
       "\n",
       "                                      sent_probs sent_label  \n",
       "0  [[2.1574432e-04 9.9863547e-01 1.1488239e-03]]          1  \n",
       "1  [[1.9508268e-04 8.0524624e-04 9.9899966e-01]]          2  \n",
       "\n",
       "[2 rows x 21 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Reading the dataset\n",
    "df_filt_2020_sent=pd.read_csv('gs://nlp_final_project_kshitijm/00_Data/NLP_FP_Data5_2020_Topics_Sentiments.csv',lineterminator='\\n')\n",
    "df_filt_2020_sent.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f30574f6-1d16-433c-a342-2321bf3e8175",
   "metadata": {},
   "source": [
    "### Re-Running on all samples"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c0406de-5a83-426a-bd2e-49ea8cf7e096",
   "metadata": {},
   "source": [
    "----\n",
    "2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c93ba483-84d3-44a7-acee-0f762d252b34",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/42868 [00:00<?, ?it/s]/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:8: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  \n",
      "100%|██████████| 42868/42868 [1:50:02<00:00,  6.49it/s]  \n",
      "100%|██████████| 42868/42868 [00:00<00:00, 300952.10it/s]\n"
     ]
    }
   ],
   "source": [
    "tqdm.pandas()\n",
    "df_filt_2021=pd.read_parquet('gs://nlp_final_project_kshitijm/00_Data/NLP_FP_Data5_2021_Topics.parquet')\n",
    "df_filt_2021['sent_probs']=df_filt_2021['cleaned_text'].progress_apply(classify_sentiment)\n",
    "df_filt_2021['sent_label']=df_filt_2021['sent_probs'].progress_apply(lambda x: np.argmax(x[0]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1252473d-74ce-47ec-8076-05609e8ca8ec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 15s, sys: 528 ms, total: 1min 15s\n",
      "Wall time: 1min 51s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_filt_2021.to_csv('gs://nlp_final_project_kshitijm/00_Data/NLP_FP_Data5_2021_Topics_Sentiments.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "3630095e-722f-4419-9930-7c25f16522e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    27175\n",
       "2    14681\n",
       "0     1012\n",
       "Name: sent_label, dtype: int64"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_filt_2021['sent_label'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd8954bf-be97-4610-b854-f57023685626",
   "metadata": {},
   "source": [
    "----\n",
    "2022"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "db7cdcd8-d403-4f3c-9bed-367559cbff33",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/54941 [00:00<?, ?it/s]/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:8: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  \n",
      "100%|██████████| 54941/54941 [2:14:11<00:00,  6.82it/s]  \n",
      "100%|██████████| 54941/54941 [00:00<00:00, 314783.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 16h 47min 41s, sys: 10min 23s, total: 16h 58min 4s\n",
      "Wall time: 2h 14min 31s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "tqdm.pandas()\n",
    "df_filt_2022=pd.read_parquet('gs://nlp_final_project_kshitijm/00_Data/NLP_FP_Data5_2022_Topics.parquet')\n",
    "df_filt_2022['sent_probs']=df_filt_2022['cleaned_text'].progress_apply(classify_sentiment)\n",
    "df_filt_2022['sent_label']=df_filt_2022['sent_probs'].progress_apply(lambda x: np.argmax(x[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "ff8c948e-c554-4e62-baf9-55c54756f60c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 30s, sys: 632 ms, total: 1min 31s\n",
      "Wall time: 2min 4s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_filt_2022.to_csv('gs://nlp_final_project_kshitijm/00_Data/NLP_FP_Data5_2022_Topics_Sentiments.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "4e0e5fe2-3525-4a0e-80a3-898b942be7ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    31806\n",
       "2    21917\n",
       "0     1218\n",
       "Name: sent_label, dtype: int64"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_filt_2022['sent_label'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bdc2d28b-d430-447f-8476-f1965b610672",
   "metadata": {},
   "source": [
    "------\n",
    "2023"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "6037db16-ca65-411f-857e-1f5425976c8c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/56339 [00:00<?, ?it/s]/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:8: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  \n",
      "100%|██████████| 56339/56339 [2:34:36<00:00,  6.07it/s]  \n",
      "100%|██████████| 56339/56339 [00:00<00:00, 310839.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 18h 45min 33s, sys: 38min 39s, total: 19h 24min 12s\n",
      "Wall time: 2h 34min 57s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "tqdm.pandas()\n",
    "df_filt_2023=pd.read_parquet('gs://nlp_final_project_kshitijm/00_Data/NLP_FP_Data5_2023_Topics.parquet')\n",
    "df_filt_2023['sent_probs']=df_filt_2023['cleaned_text'].progress_apply(classify_sentiment)\n",
    "df_filt_2023['sent_label']=df_filt_2023['sent_probs'].progress_apply(lambda x: np.argmax(x[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "292d5ce2-ef08-4f2d-9519-63adcb7f2bb5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 46s, sys: 801 ms, total: 1min 47s\n",
      "Wall time: 2min 21s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_filt_2023.to_csv('gs://nlp_final_project_kshitijm/00_Data/NLP_FP_Data5_2023_Topics_Sentiments.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "c96e4aa4-0ce0-44fd-869b-170c1530a0fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    39723\n",
       "2    14854\n",
       "0     1762\n",
       "Name: sent_label, dtype: int64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_filt_2023['sent_label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "4a82c9c7-bc95-4430-8cdb-984e58dff00f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>clean_title</th>\n",
       "      <th>sent_probs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19164</th>\n",
       "      <td>Sexting chatbot ban points to looming battle over AI rules</td>\n",
       "      <td>[[0.81558263, 0.18129273, 0.003124689]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10530</th>\n",
       "      <td>Can AI revive the handset market?</td>\n",
       "      <td>[[0.9969887, 0.0023647698, 0.00064651243]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42106</th>\n",
       "      <td>Why Developers Are Turning to AI to Deliver Complex Projects</td>\n",
       "      <td>[[0.9829107, 0.016453687, 0.0006356551]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21248</th>\n",
       "      <td>\\r Naver to introduce search GPT in first half of year</td>\n",
       "      <td>[[0.9225757, 0.07364595, 0.003778383]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15365</th>\n",
       "      <td>Google's new AI bot made a gaffe because it was 'rushed' out to compete with ChatGPT, experts say</td>\n",
       "      <td>[[0.99728346, 0.0020851572, 0.00063134957]]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                              clean_title  \\\n",
       "19164                                          Sexting chatbot ban points to looming battle over AI rules   \n",
       "10530                                                                   Can AI revive the handset market?   \n",
       "42106                                        Why Developers Are Turning to AI to Deliver Complex Projects   \n",
       "21248                                              \\r Naver to introduce search GPT in first half of year   \n",
       "15365  Google's new AI bot made a gaffe because it was 'rushed' out to compete with ChatGPT, experts say    \n",
       "\n",
       "                                        sent_probs  \n",
       "19164      [[0.81558263, 0.18129273, 0.003124689]]  \n",
       "10530   [[0.9969887, 0.0023647698, 0.00064651243]]  \n",
       "42106     [[0.9829107, 0.016453687, 0.0006356551]]  \n",
       "21248       [[0.9225757, 0.07364595, 0.003778383]]  \n",
       "15365  [[0.99728346, 0.0020851572, 0.00063134957]]  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.options.display.max_colwidth=None\n",
    "df_filt_2023[df_filt_2023['sent_label']==0][['clean_title','sent_probs']].sample(5)"
   ]
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "python3",
   "name": "tf2-gpu.2-11.m108",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-11:m108"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}