reddit_jokes_upvote_ratio_prediction_classifier_3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/ali/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using device: cuda\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "----------\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d86169c27c946b29e0aec96bf8e6fbd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training:   0%|          | 0/1875 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss 0.655776778459549 accuracy 0.6129860601614087\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c9bc5016d19a46aeb0e240815da4e950",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating:   0%|          | 0/466 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val   loss 0.6697926031417601 accuracy 0.5990863898965472\n",
      "\n",
      "Epoch 2/3\n",
      "----------\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07bb569612af4c0f9990ec8397ddba4e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training:   0%|          | 0/1875 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss 0.5218678290685018 accuracy 0.7442806643100113\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ca5cc20de3f4cf99e3fb9d1005a0a64",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating:   0%|          | 0/466 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val   loss 0.8020383442228444 accuracy 0.6083568453580546\n",
      "\n",
      "Epoch 3/3\n",
      "----------\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "92eb66086b40426e89429cd483d087ee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training:   0%|          | 0/1875 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss 0.37137402694622673 accuracy 0.8414926965917428\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3f7b1a8982b4af997d367f83616fc9d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Evaluating:   0%|          | 0/466 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val   loss 0.9234421154715984 accuracy 0.613999731291146\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.61      0.64      0.63      3765\n",
      "           1       0.62      0.58      0.60      3678\n",
      "\n",
      "    accuracy                           0.61      7443\n",
      "   macro avg       0.61      0.61      0.61      7443\n",
      "weighted avg       0.61      0.61      0.61      7443\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from transformers import BertTokenizer, BertForSequenceClassification, get_linear_schedule_with_warmup\n",
    "from torch.optim import AdamW\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "from sklearn.utils import resample\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "\n",
    "# Download NLTK resources if not already present\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "# Function to clean text\n",
    "def clean_text(text):\n",
    "    text = text.lower()\n",
    "    text = re.sub(r\"[^a-zA-Z0-9]\", \" \", text)\n",
    "    text = re.sub(r\"\\s+\", \" \", text)\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    text = ' '.join(word for word in text.split() if word not in stop_words)\n",
    "    return text\n",
    "\n",
    "# Load dataset\n",
    "file_path = 'data/reddit_jokes_slim_processed.csv'  # Update with your file path\n",
    "df = pd.read_csv(file_path)\n",
    "\n",
    "# Clean text and split data\n",
    "df['thread_selftext_clean'] = df['thread_selftext'].apply(clean_text)\n",
    "df['thread_title_clean'] = df['thread_title'].apply(clean_text)\n",
    "df['cleaned_text'] = df['thread_title_clean'] + ' ' + df['thread_selftext_clean']\n",
    "\n",
    "median_value = df['thread_upvote_ratio'].median()\n",
    "df['binary_class'] = (df['thread_upvote_ratio'] > median_value).astype(int)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(df['cleaned_text'], df['binary_class'], test_size=0.2, random_state=42)\n",
    "\n",
    "# Combine them back for resampling\n",
    "train_data = pd.concat([X_train, y_train], axis=1)\n",
    "\n",
    "# Separate majority and minority classes\n",
    "majority_class = train_data[train_data.binary_class==0]\n",
    "minority_class = train_data[train_data.binary_class==1]\n",
    "\n",
    "# Upsample minority class\n",
    "minority_class_upsampled = resample(minority_class,\n",
    "                                   replace=True, # sample with replacement\n",
    "                                   n_samples=len(majority_class), # to match majority class\n",
    "                                   random_state=42) # reproducible results\n",
    "\n",
    "# Combine majority class with upsampled minority class\n",
    "upsampled_train_data = pd.concat([majority_class, minority_class_upsampled])\n",
    "\n",
    "# Shuffle the dataset to prevent the model from learning anything from the order of the examples\n",
    "upsampled_train_data = upsampled_train_data.sample(frac=1, random_state=42)\n",
    "\n",
    "# Split upsampled data into X_train and y_train\n",
    "X_train = upsampled_train_data['cleaned_text']\n",
    "y_train = upsampled_train_data['binary_class']\n",
    "# Compute class weights\n",
    "class_weights = compute_class_weight('balanced', classes=np.unique(y_train), y=y_train)\n",
    "class_weights = torch.tensor(class_weights, dtype=torch.float).to(device)\n",
    "\n",
    "loss_fn = nn.CrossEntropyLoss(weight=class_weights)\n",
    "\n",
    "# BERT Tokenizer\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "\n",
    "class JokesDataset(Dataset):\n",
    "    def __init__(self, texts, labels, tokenizer, max_len):\n",
    "        self.texts = texts\n",
    "        self.labels = labels\n",
    "        self.tokenizer = tokenizer\n",
    "        self.max_len = max_len\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.texts)\n",
    "    \n",
    "    def __getitem__(self, item):\n",
    "        text = str(self.texts[item])\n",
    "        label = self.labels[item]\n",
    "\n",
    "        encoding = self.tokenizer.encode_plus(\n",
    "            text,\n",
    "            add_special_tokens=True,\n",
    "            max_length=self.max_len,\n",
    "            return_token_type_ids=False,\n",
    "            padding='max_length',\n",
    "            return_attention_mask=True,\n",
    "            return_tensors='pt',\n",
    "            truncation=True\n",
    "        )\n",
    "\n",
    "        return {\n",
    "            'text': text,\n",
    "            'input_ids': encoding['input_ids'].flatten(),\n",
    "            'attention_mask': encoding['attention_mask'].flatten(),\n",
    "            'labels': torch.tensor(int(label), dtype=torch.long)\n",
    "        }\n",
    "\n",
    "# Create dataset objects\n",
    "max_len = 128\n",
    "batch_size = 16\n",
    "train_data = JokesDataset(X_train.to_numpy(), y_train.to_numpy(), tokenizer, max_len=128)\n",
    "train_loader = DataLoader(train_data, batch_size=16, num_workers=2)\n",
    "test_data = JokesDataset(X_test.to_numpy(), y_test.to_numpy(), tokenizer, max_len=128)\n",
    "test_loader = DataLoader(test_data, batch_size=16, num_workers=2)\n",
    "\n",
    "# Setting device\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"Using device:\", device)\n",
    "\n",
    "# Load pre-trained BERT model\n",
    "model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2).to(device)\n",
    "\n",
    "# Freeze all layers in the BERT model\n",
    "#for param in model.bert.parameters():\n",
    "#    param.requires_grad = False\n",
    "\n",
    "# Unfreeze the classifier layer (final layer)\n",
    "#for param in model.classifier.parameters():\n",
    "#    param.requires_grad = True\n",
    "\n",
    "#Setting up training parameters\n",
    "epochs = 3\n",
    "#optimizer = AdamW(model.classifier.parameters(), lr=2e-5)\n",
    "optimizer = AdamW(model.parameters(), lr=2e-5)\n",
    "total_steps = len(train_loader) * epochs\n",
    "scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=total_steps)\n",
    "\n",
    "# Training Function\n",
    "def train_epoch(model, data_loader, optimizer, device, scheduler, n_examples,loss_fn):\n",
    "    model.train()\n",
    "    losses = []\n",
    "    correct_predictions = 0\n",
    "    tqdm_data_loader = tqdm(data_loader, total=len(data_loader), desc=\"Training\", leave=False)\n",
    "\n",
    "    for d in tqdm_data_loader:\n",
    "        input_ids = d[\"input_ids\"].to(device)\n",
    "        attention_mask = d[\"attention_mask\"].to(device)\n",
    "        labels = d[\"labels\"].to(device)\n",
    "\n",
    "        outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=labels)\n",
    "        loss = outputs.loss\n",
    "        logits = outputs.logits\n",
    "        _, preds = torch.max(logits, dim=1)\n",
    "        correct_predictions += torch.sum(preds == labels)\n",
    "        losses.append(loss.item())\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n",
    "        optimizer.step()\n",
    "        scheduler.step()\n",
    "\n",
    "        tqdm_data_loader.set_postfix(loss=loss.item())\n",
    "\n",
    "    return correct_predictions.double() / n_examples, np.mean(losses)\n",
    "\n",
    "\n",
    "\n",
    "# Evaluation Function\n",
    "def eval_model(model, data_loader, device, n_examples,loss_fn):\n",
    "    model.eval()\n",
    "    losses = []\n",
    "    correct_predictions = 0\n",
    "    tqdm_data_loader = tqdm(data_loader, desc=\"Evaluating\", leave=False)\n",
    "\n",
    "    for d in tqdm_data_loader:\n",
    "        input_ids = d[\"input_ids\"].to(device)\n",
    "        attention_mask = d[\"attention_mask\"].to(device)\n",
    "        labels = d[\"labels\"].to(device)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=labels)\n",
    "            loss = outputs.loss\n",
    "            logits = outputs.logits\n",
    "            _, preds = torch.max(logits, dim=1)\n",
    "            correct_predictions += torch.sum(preds == labels)\n",
    "            losses.append(loss.item())\n",
    "\n",
    "    return correct_predictions.double() / n_examples, np.mean(losses)\n",
    "\n",
    "\n",
    "#Training and Evaluation\n",
    "for epoch in range(epochs):\n",
    "    print(f'Epoch {epoch + 1}/{epochs}')\n",
    "    print('-' * 10)\n",
    "    train_acc, train_loss = train_epoch(\n",
    "    model, \n",
    "    train_loader, \n",
    "    optimizer, \n",
    "    device, \n",
    "    scheduler, \n",
    "    len(train_data),\n",
    "    loss_fn\n",
    "    )\n",
    "    print(f'Train loss {train_loss} accuracy {train_acc}')\n",
    "\n",
    "    val_acc, val_loss = eval_model(\n",
    "    model, \n",
    "    test_loader, \n",
    "    device, \n",
    "    len(test_data),\n",
    "    loss_fn\n",
    "    )\n",
    "    print(f'Val   loss {val_loss} accuracy {val_acc}')\n",
    "    print()\n",
    "\n",
    "y_pred = []\n",
    "y_true = []\n",
    "\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    for d in test_loader:\n",
    "        input_ids = d[\"input_ids\"].to(device)\n",
    "        attention_mask = d[\"attention_mask\"].to(device)\n",
    "        labels = d[\"labels\"].to(device)\n",
    "        outputs = model(input_ids=input_ids, attention_mask=attention_mask)\n",
    "        _, preds = torch.max(outputs.logits, dim=1)\n",
    "\n",
    "        y_pred.extend(preds.detach().cpu().numpy())\n",
    "        y_true.extend(labels.detach().cpu().numpy())\n",
    "\n",
    "    print(classification_report(y_true, y_pred))\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ds_ve_39_DS",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}