4_TDA_acceptability_classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from time import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import make_scorer, matthews_corrcoef, accuracy_score, balanced_accuracy_score, roc_curve\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.model_selection import StratifiedKFold,GridSearchCV\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_selection import VarianceThreshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "K2dHzOal8ac8"
   },
   "source": [
    "# Read features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.read_features import *\n",
    "from src.threshold_search import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dSFcFafeBKYW",
    "outputId": "21615155-8acf-43fe-aa27-1a562bf0eac3"
   },
   "outputs": [],
   "source": [
    "STATE=42\n",
    "model_dir = \"./bert-base-cased-en-cola_32_3e-05_lr_0.01_decay_balanced/\"\n",
    "data_dir = \"./data/en-cola/\"\n",
    "heads = 16 if \"roberta\" in model_dir.lower() else 12\n",
    "layers = 24 if \"roberta\" in model_dir.lower() else 12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "ucQ5QZPaloet"
   },
   "outputs": [],
   "source": [
    "file_type = \".csv\" # .csv or .tsv\n",
    "train_set_name, valid_set_name, test_set_name = (\"train\", \"dev\",\"test\") \n",
    "data_args = dict(((k, eval(k)) for k in (\"data_dir\", \"file_type\")))\n",
    "(sents_train, y_train), (sents_valid, y_valid), (sents_test, y_test) = list(map(lambda x_: read_labels(x_, **data_args), \n",
    "                                                [x_ for x_ in (train_set_name, valid_set_name, test_set_name)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "9mZFHPip4CWc"
   },
   "outputs": [],
   "source": [
    "topological_thr = 6\n",
    "features_dir = model_dir + \"/features/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1YLVJrbO2R2c",
    "outputId": "3b4d4324-2612-4fc1-c9ae-f771837eedcb"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading train features...: 100%|██████████| 432/432 [00:18<00:00, 23.10it/s]\n",
      "Loading dev features...: 100%|██████████| 432/432 [00:02<00:00, 176.11it/s]\n",
      "Loading test features...: 100%|██████████| 432/432 [00:02<00:00, 176.37it/s]\n"
     ]
    }
   ],
   "source": [
    "kwargs = dict(((k, eval(k)) for k in (\"features_dir\", \"model_dir\", \"topological_thr\")))\n",
    "kwargs[\"heads\"] = heads\n",
    "kwargs[\"layers\"] =layers\n",
    "X_train, X_valid, X_test = list(map(lambda x_: load_features(x_, **kwargs), [x_ for x_ in (train_set_name, valid_set_name, test_set_name)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "FMvm9ZIirgNs"
   },
   "outputs": [],
   "source": [
    "# # Exclude weakly connected components equal to b0 Betti number\n",
    "X_train = X_train.iloc[:, ~X_train.columns.str.startswith('w')]\n",
    "X_valid = X_valid.loc[:, X_train.columns]\n",
    "X_test = X_test.loc[:, X_train.columns]\n",
    "\n",
    "# # # # Removing constant and quasi-constant features\n",
    "var_thr = VarianceThreshold(threshold = 0.00001)\n",
    "var_thr.fit(X_valid)\n",
    "not_constant_f = var_thr.get_support()\n",
    "X_train = X_train.loc[:, not_constant_f]\n",
    "X_valid = X_valid.loc[:, not_constant_f]\n",
    "X_test = X_test.loc[:, not_constant_f]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(527, 8800)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([\"Our friends won't buy this analysis, let alone the next one we propose.\",\n",
       "       \"One more pseudo generalization and I'm giving up.\",\n",
       "       \"One more pseudo generalization or I'm giving up.\", ...,\n",
       "       'It is easy to slay the Gorgon.',\n",
       "       'I had the strangest feeling that I knew you.',\n",
       "       'What all did you get for Christmas?'], dtype=object)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sents_train.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Acceptability judgements with PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def score_mcc(y_true, y_pred):\n",
    "    return matthews_corrcoef(y_true, y_pred)\n",
    "score_mcc_ = make_scorer(score_mcc, greater_is_better=True)\n",
    "# Print summary statistics of the results\n",
    "def report(results, n_top=3):\n",
    "    for i in range(1, n_top + 1):\n",
    "        candidates = np.flatnonzero(results['rank_test_score'] == i)\n",
    "        for candidate in candidates:\n",
    "            print(\"Model with rank: {0}\".format(i))\n",
    "            print(\"Mean validation score: {0:.3f} (std: {1:.3f})\"\n",
    "                  .format(results['mean_test_score'][candidate],\n",
    "                          results['std_test_score'][candidate]))\n",
    "            print(\"Parameters: {0}\".format(results['params'][candidate]))\n",
    "            print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((8551, 8800), (8551,))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 24 candidates, totalling 72 fits\n",
      "GridSearchCV took 88.50 seconds for 24 candidate parameter settings.\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.971 (std: 0.002)\n",
      "Parameters: {'clf__C': 0.1, 'clf__class_weight': None, 'clf__max_iter': 25, 'reduce_dim__n_components': 250}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.971 (std: 0.002)\n",
      "Parameters: {'clf__C': 0.1, 'clf__class_weight': None, 'clf__max_iter': 100, 'reduce_dim__n_components': 250}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.971 (std: 0.003)\n",
      "Parameters: {'clf__C': 0.1, 'clf__class_weight': None, 'clf__max_iter': 25, 'reduce_dim__n_components': 200}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.971 (std: 0.003)\n",
      "Parameters: {'clf__C': 0.1, 'clf__class_weight': None, 'clf__max_iter': 100, 'reduce_dim__n_components': 200}\n",
      "\n",
      "Model with rank: 5\n",
      "Mean validation score: 0.970 (std: 0.002)\n",
      "Parameters: {'clf__C': 0.1, 'clf__class_weight': None, 'clf__max_iter': 25, 'reduce_dim__n_components': 100}\n",
      "\n",
      "Model with rank: 5\n",
      "Mean validation score: 0.970 (std: 0.002)\n",
      "Parameters: {'clf__C': 0.1, 'clf__class_weight': None, 'clf__max_iter': 100, 'reduce_dim__n_components': 100}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Parameters grid\n",
    "params = {'tol': 1e-6, 'random_state': STATE, 'solver': 'liblinear', \"penalty\": 'l1'}\n",
    "N_FEATURES_OPTIONS =  np.arange(100,300,50)\n",
    "C_OPTIONS = [1e-3, 0.01, 0.1]\n",
    "CLASS_WEIGHT = [None]\n",
    "#'reduce_dim__n_components': N_FEATURES_OPTIONS,\n",
    "from sklearn.preprocessing import *\n",
    "all_preprocessors = [\n",
    "    StandardScaler(),QuantileTransformer(n_quantiles=100), MinMaxScaler(), RobustScaler()\n",
    "]\n",
    "# all_preprocessors = [QuantileTransformer(n_quantiles=100)]\n",
    "max_iter_range = [25,100]\n",
    "#'clf__max_iter':max_iter_range, \n",
    "params_grid = {'reduce_dim__n_components': N_FEATURES_OPTIONS,\n",
    "               'clf__max_iter':max_iter_range, 'clf__C': C_OPTIONS,\n",
    "               'clf__class_weight': CLASS_WEIGHT, \n",
    "               }# 'scaler':all_preprocessors 'scaler':all_preprocessors\n",
    "pipeline = Pipeline([\n",
    "    ('scaler', StandardScaler()),\n",
    "    ('reduce_dim', PCA(whiten=True, random_state=STATE)),\n",
    "    ('clf', LogisticRegression(**params))])\n",
    "kfold = StratifiedKFold(n_splits=3, shuffle=True, random_state=STATE)\n",
    "ids=(np.arange(0, X_train.shape[0]), np.arange(X_train.shape[0], X_train.shape[0]+X_valid.shape[0]))\n",
    "X=np.concatenate((X_train, X_valid), axis=0)\n",
    "y=np.concatenate((y_train, y_valid), axis=0)\n",
    "clf_ = GridSearchCV(pipeline, cv=kfold, verbose=4, param_grid=params_grid, scoring=\"accuracy\", n_jobs=30)\n",
    "start = time()\n",
    "clf_.fit(X_train, y_train)\n",
    "print(\"GridSearchCV took %.2f seconds for %d candidate parameter settings.\"\n",
    "      % (time() - start, len(clf_.cv_results_['params'])))\n",
    "report(clf_.cv_results_, n_top=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bal. Acc. =\t0.847\n",
      "Accuracy =\t0.856\n",
      "MCC score =\t0.649\t\n",
      "\n",
      "Bal. Acc. =\t0.79\n",
      "Accuracy =\t0.806\n",
      "MCC score =\t0.527\t\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8062015503875969, 0.5270938469686087)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print_scores(clf_.best_estimator_.predict(X_valid), y_valid)\n",
    "print_scores(clf_.best_estimator_.predict(X_test), y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bal. Acc. =\t0.847\n",
      "Accuracy =\t0.856\n",
      "MCC score =\t0.649\t\n",
      "\n",
      "Bal. Acc. =\t0.79\n",
      "Accuracy =\t0.806\n",
      "MCC score =\t0.527\t\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8062015503875969, 0.5270938469686087)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print_scores(clf_.best_estimator_.predict(X_valid), y_valid)\n",
    "print_scores(clf_.best_estimator_.predict(X_test), y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search for the best threshold by maximizing \"balanced_accuracy\":\n",
      "Threshold =\t0.8153923144290629\n",
      "Bal. Acc. =\t0.9759274245665875\n",
      "Accuracy =\t0.973687288036487\n",
      "MCC score =\t0.9383722483301667\t\n",
      "\n",
      "Search for the best threshold by maximizing \"matthews_corrcoef\":\n",
      "Threshold =\t0.5575001697909184\n",
      "Bal. Acc. =\t0.9724230901796279\n",
      "Accuracy =\t0.976026195766577\n",
      "MCC score =\t0.942590731097539\t\n",
      "\n",
      "Bal. Acc. =\t0.803\n",
      "Accuracy =\t0.856\n",
      "MCC score =\t0.649\t\n",
      "\n",
      "Bal. Acc. =\t0.749\n",
      "Accuracy =\t0.812\n",
      "MCC score =\t0.543\t\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.812015503875969, 0.5427800325174282)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yhat_train = clf_.best_estimator_.predict_proba(X_train)[:, 1]\n",
    "thr_mcc, _, _ = print_thresholds(y_train, yhat_train)\n",
    "yhat_valid = clf_.best_estimator_.predict_proba(X_valid)[:, 1]\n",
    "y_pred_thr_mcc_valid = np.where(yhat_valid >= thr_mcc, 1, 0)\n",
    "print_scores(y_valid, y_pred_thr_mcc_valid)\n",
    "yhat_test = clf_.best_estimator_.predict_proba(X_test)[:, 1]\n",
    "y_pred_thr_mcc = np.where(yhat_test >= thr_mcc, 1, 0)\n",
    "print_scores(y_test, y_pred_thr_mcc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search for the best threshold by maximizing \"balanced_accuracy\":\n",
      "Threshold =\t0.8483109363718219\n",
      "Bal. Acc. =\t0.8423135464231355\n",
      "Accuracy =\t0.8671726755218216\n",
      "MCC score =\t0.6870130672514878\t\n",
      "\n",
      "Search for the best threshold by maximizing \"matthews_corrcoef\":\n",
      "Threshold =\t0.8483109363718219\n",
      "Bal. Acc. =\t0.8423135464231355\n",
      "Accuracy =\t0.8671726755218216\n",
      "MCC score =\t0.6870130672514878\t\n",
      "\n",
      "Bal. Acc. =\t0.773\n",
      "Accuracy =\t0.81\n",
      "MCC score =\t0.553\t\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.810077519379845, 0.5534967168498809)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yhat_train = clf_.best_estimator_.predict_proba(X_valid)[:, 1]\n",
    "thr_mcc, _, _ = print_thresholds(y_valid, yhat_train)\n",
    "yhat_test = clf_.best_estimator_.predict_proba(X_test)[:, 1]\n",
    "y_pred_thr_mcc = np.where(yhat_test >= thr_mcc, 1, 0)\n",
    "print_scores(y_test, y_pred_thr_mcc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import joblib\n",
    "# joblib.dump(best_pipe, 'grid_search_bert_trained_en_tda_all.pkl')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}