3. Model Building.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ddefcc9b",
   "metadata": {},
   "source": [
    "# Whisky Project\n",
    "\n",
    "## 3. Model Building\n",
    "\n",
    "### Creating and Tuning ML Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "55214024",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\jamie\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "# import libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import time\n",
    "import re\n",
    "import joblib \n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk.corpus import wordnet, stopwords\n",
    "nltk.download('stopwords')\n",
    "\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
    "from sklearn.model_selection import train_test_split, RandomizedSearchCV, GridSearchCV\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.linear_model import LogisticRegression, SGDClassifier\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.gaussian_process import GaussianProcessClassifier\n",
    "from sklearn.gaussian_process.kernels import RBF\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c4a56dc3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1985, (27513, 7), (18868, 7))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read in data and tasting_note_list\n",
    "df = pd.read_csv('./data/branded.csv', index_col = 'Unnamed: 0')\n",
    "df_sms = pd.read_csv('./data/branded_singlemalts.csv', index_col = 'Unnamed: 0')\n",
    "\n",
    "with open('./data/tasting_notes.txt', 'r') as f:\n",
    "    wordlist = [item[:-1] for item in f.readlines()]\n",
    "\n",
    "len(wordlist), df.shape, df_sms.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4834a6bd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataframe df had 27513 rows. After dropping rows with 10 or fewer reviews, it now has 26953 rows.\n",
      "Dataframe df_sms had 18868 rows. After dropping rows with 10 or fewer reviews, it now has 18780 rows.\n"
     ]
    }
   ],
   "source": [
    "# dropping brands that only have a frequency of lower_limit or fewer because it screws\n",
    "# up cross-validation.\n",
    "def drop_unpopular_brands(name, df, lower_limit=0):\n",
    "    print(\"Dataframe {} had {} rows. After dropping rows with {} or fewer reviews, \".format(\n",
    "        name, df.shape[0], lower_limit), end=\"\")\n",
    "    dfbrandcounts = df['brand'].value_counts()\n",
    "    limited_list = dfbrandcounts[dfbrandcounts>lower_limit].index\n",
    "    df = df[df['brand'].isin(limited_list)]\n",
    "    print(\"it now has {} rows.\".format(df.shape[0]))\n",
    "    return df\n",
    "\n",
    "lower_limit = 10\n",
    "df = drop_unpopular_brands('df', df, 10)\n",
    "df_sms = drop_unpopular_brands('df_sms', df_sms, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "757e044c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27045    Laphroaig\n",
       "27046    Laphroaig\n",
       "27050    Laphroaig\n",
       "27054    Laphroaig\n",
       "27063    Laphroaig\n",
       "           ...    \n",
       "28059    Laphroaig\n",
       "28209    Laphroaig\n",
       "28261    Laphroaig\n",
       "28308    Laphroaig\n",
       "28370    Laphroaig\n",
       "Name: brand, Length: 1009, dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.brand[df.brand=='Laphroaig']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3203d20c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenise_and_stem_text(text):\n",
    "    '''\n",
    "    INPUTS:\n",
    "    text (string) - what you want to be lemmatised\n",
    "    OUTPUTS:\n",
    "    lemmas (list) - list of lemmatised next\n",
    "    '''\n",
    "    # Import stopword list and update with a few of my own\n",
    "    stopword_list = stopwords.words(\"english\")\n",
    "    [stopword_list.append(i) for i in ['nose', 'palate', 'taste', 'finish']]\n",
    "    \n",
    "    # Normalise text - remove numbers too as we don't need them\n",
    "    text = re.sub(r\"[^a-zA-Z]\", \" \", text.lower())\n",
    "    \n",
    "    # tokenise\n",
    "    words = text.split()\n",
    "    \n",
    "    # Checks it's a word and removes stop words\n",
    "    words = [word for word in words if word not in stopword_list]\n",
    "    \n",
    "    # Create stemmer object\n",
    "    stemmer = PorterStemmer()\n",
    "    \n",
    "    # Add lemmas\n",
    "    lemmas = []\n",
    "    for word in words:\n",
    "        lemmas.append(stemmer.stem(word))\n",
    "    \n",
    "    return lemmas"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a79f4896",
   "metadata": {},
   "source": [
    "---\n",
    "#### Initial attempt\n",
    "\n",
    "What follows here was my initial attempt to create a tokeniser and classifier of the whiskies. For some reason, I got it into my head that I needed to use the custom vocabulary list and apply a vectoriser for nose, palate and finish, and then start reshaping the sparse matrices that came out of it by deleting empty columns. I later realised that this indicates a misunderstanding of what a sparse matrix is and how it is handled, and thus this was all a waste of time that I couldn't get to work inside a pipeline anyway, but I've included it here as it's somewhat interesting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f1d465bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#vectoriser_with_wordlist = CountVectorizer(tokenizer=tokenise_and_stem_text, vocabulary=wordlist)\n",
    "\n",
    "#X_nose = vectoriser_with_wordlist.fit_transform(df['nose'])\n",
    "#X_palate = vectoriser_with_wordlist.fit_transform(df['palate'])\n",
    "#X_finish = vectoriser_with_wordlist.fit_transform(df['finish'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ec5a4f11",
   "metadata": {},
   "outputs": [],
   "source": [
    "#for i in range(50):\n",
    "#    print(\"There are {} columns in the Nose matrix with {} '1's in it.\".format(\n",
    "#    (X_nose[X_nose.sum(axis=0)==i]).shape[1], i))\n",
    "#    print(\"There are {} columns in the Palate matrix with {} '1's in it.\".format(\n",
    "#    (X_palate[X_palate.sum(axis=0)==i]).shape[1], i))\n",
    "#   print(\"There are {} columns in the Finish matrix with {} '1's in it.\".format(\n",
    "#   (X_finish[X_finish.sum(axis=0)==i]).shape[1], i))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "443daf4b",
   "metadata": {},
   "source": [
    "Here I realised that you can dramatically cut down the size of the matrix by dropping any columns that have fewer than 4 hits in them. But even using a lower bound of 2 would substantially reduce their size. \n",
    "\n",
    "Again, this shows something of a misconception. Unique hits like this tend to be informationally _more_ valuable than the opposite. After all, the point is to discriminate, and something that uniquely picks out one whisky is fairly useful."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "321cc3e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_smaller_matrices(matrix, lowerbound=0, wordlist=wordlist):\n",
    "    '''\n",
    "    INPUTS:\n",
    "    matrix - matrix you wish to make smaller \n",
    "    lowerbound - inclusive. The smallest total number of hits in the column of the matrix that\n",
    "                you'll allow. e.g. if there's only 1 review that mentions 'ablaze',\n",
    "                the sum for axis=0 will be 1. Worth keeping? Probably not...\n",
    "    wordlist - list of words that are indexed in the same way as matrix\n",
    "    OUTPUTS:\n",
    "    smaller_matrix - smaller matrix to use \n",
    "    smaller_list - smaller list of words.\n",
    "    '''\n",
    "    # Get indices of the columns we want to keep (i.e. where col sums >= lowerbound)\n",
    "    row_index, col_index = np.where(matrix.sum(axis=0)>=lowerbound)\n",
    "        \n",
    "    smaller_list = []\n",
    "    for i in range(len(col_index)):\n",
    "        column_to_add = matrix.toarray()[:,col_index[i]].copy().reshape(-1, 1)\n",
    "        if i == 0:\n",
    "            # Create new_matrix with first column\n",
    "            smaller_matrix = column_to_add\n",
    "        else:\n",
    "            # Concatenate new_matrix with i'th column. \n",
    "            smaller_matrix = np.hstack((smaller_matrix, column_to_add))\n",
    "           \n",
    "        # Add to newlist in order they appear in wordlist\n",
    "        smaller_list.append(wordlist[col_index[i]])\n",
    "        \n",
    "    return smaller_matrix, smaller_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6789572f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create smaller matrices for each in turn with a lowerbound of 3. \n",
    "#lb = 3\n",
    "#nose_matrix, nose_list = create_smaller_matrices(X_nose, lowerbound=lb, wordlist=wordlist)\n",
    "#palate_matrix, palate_list = create_smaller_matrices(X_palate, lowerbound=lb, wordlist=wordlist)\n",
    "#finish_matrix, finish_list = create_smaller_matrices(X_finish, lowerbound=lb, wordlist=wordlist)\n",
    "\n",
    "# Overlapping areas in terms of tasting notes between the three domains:\n",
    "#overlap_list = sorted((set(nose_list) & set(palate_list)) & set(finish_list))\n",
    "#nose_matrix.shape[1], palate_matrix.shape[1], finish_matrix.shape[1], len(overlap_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "25f45190",
   "metadata": {},
   "outputs": [],
   "source": [
    "#nose_list = ['nose_'+item for item in nose_list]\n",
    "#nose_list"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "144cc8d7",
   "metadata": {},
   "source": [
    "My comment at the time was:\n",
    "\n",
    ">Now I'm finally able to produce a matrix of tasting notes, separated by nose, palate and finish sections for each row of the df_collated dataframe. The overlap list might be helpful down the line if I want to perform analyses along the lines of 'Which whiskies are the most different from nose to palate?' or similar. The next tasks I need to perform are to hstack these matrices together to create a matrix of shape 27513 x 3052 (or whatever it is, depending on the hyperparameters used), and keep track of what each column means. Then we need to start training a Classifier using the matrix. \n",
    "\n",
    ">At this point, I need to write this as a proper function and put it inside a pipeline, because I'll need to split the data into a training and testing set in order to evaluate different potential classifiers. Technically, I already have a bit of data leakage here because I constructed my vocublary list using the full dataset. However, it's not a serious leakage as I was mainly just including as many tasting-y sounding words as possible in that wordlist. Since very rarely used words aren't going to be that important anyway, it's extremely unlikely that a vocab list generated from 20,000 reviews (say) as opposed to 25,000 would be that different.\n",
    "\n",
    "The function below was intended simply to concatenate the reduced matrices for nose, palate and finish together, and keep track of what each column meant by creating a 'tasting note list'. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6a70107d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_review_by_tasting_note_matrix(lowerbound=0):\n",
    "    \n",
    "    # create smaller matrices for each in turn with a lowerbound set to lowerbound. \n",
    "    lb = lowerbound\n",
    "    nose_matrix, nose_list = create_smaller_matrices(X_nose, lowerbound=lb, wordlist=wordlist)\n",
    "    palate_matrix, palate_list = create_smaller_matrices(X_palate, lowerbound=lb, wordlist=wordlist)\n",
    "    finish_matrix, finish_list = create_smaller_matrices(X_finish, lowerbound=lb, wordlist=wordlist)\n",
    "    \n",
    "    # Overlapping areas in terms of tasting notes between the three domains:\n",
    "    overlap_list = sorted((set(nose_list) & set(palate_list)) & set(finish_list))\n",
    "    \n",
    "    # Hstack the three matrices\n",
    "    review_by_tasting_note_matrix = np.hstack((nose_matrix, palate_matrix, finish_matrix))\n",
    "    \n",
    "    # Create tasting_note_list\n",
    "    nose_list = ['nose_'+item for item in nose_list]\n",
    "    palate_list = ['palate_'+item for item in palate_list]\n",
    "    finish_list = ['finish_'+item for item in palate_list]\n",
    "    tasting_note_list = nose_list+palate_list+finish_list\n",
    "    \n",
    "    return review_by_tasting_note_matrix, tasting_note_list, overlap_list\n",
    "\n",
    "# review_by_tasting_note_matrix, tasting_note_list, overlap_list = create_review_by_tasting_note_matrix(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0509f7f",
   "metadata": {},
   "source": [
    "At this point I realised this wasn't the right approach:\n",
    "\n",
    "1. You shouldn't be eliminating columns with a few hits in them anyway.\n",
    "2. Even deleting empty columns is basically a waste of time because of the way sparse matrices are handled by sklearn and scipy.\n",
    "3. I needed to write a pipeline so that I could start trying out classifiers, using cross-validation and fine-tuning, etc., and my functions weren't going to play nice inside a pipeline.\n",
    "\n",
    "---\n",
    "#### Model Implementation (New Approach)\n",
    "\n",
    "I want to set up a pipeline now without any of this matrix reduction stuff, but simply employing three count-vectorisers in parallel for the nose, palate and finish respectively. I don't think there's going to be any way here to keep track of what each column means, but that's OK. Once the model is created and tuned and we know exactly what we want, we can always go back and build it manually and interpret it.\n",
    "\n",
    "So we need three tiny, bespoke functions that simply grabs the correct column of data. Then we run three countvectorizers in parallel, then we bring them together using featureunion, run them through a TfIDF transformer, and then a classifier.\n",
    "\n",
    "Below is an earlier pipeline I used to try out various options in the design of the pipeline:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "91cb64a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper functions for getting the right column of data to use in the pipeline:\n",
    "def getnose(array):\n",
    "    return array[:,0]\n",
    "    \n",
    "def getpalate(array):\n",
    "    return array[:,1]\n",
    "\n",
    "def getfinish(array):\n",
    "    return array[:,2]\n",
    "\n",
    "# Earlier pipeline I used to try out different classifiers.\n",
    "pipeline = Pipeline([\n",
    "        \n",
    "        ('union', FeatureUnion([\n",
    "            \n",
    "            ('pipe_nose', Pipeline([\n",
    "                ('get_nose', FunctionTransformer(getnose)),\n",
    "                #('countvec_nose', CountVectorizer(\n",
    "                #    tokenizer=tokenise_and_stem_text) )\n",
    "                ('countvec_nose', CountVectorizer(\n",
    "                    tokenizer=tokenise_and_stem_text, vocabulary=wordlist) )\n",
    "            ]) ),\n",
    "            ('pipe_palate', Pipeline([\n",
    "                ('get_palate', FunctionTransformer(getpalate)),\n",
    "                #('countvec_palate', CountVectorizer(\n",
    "                #    tokenizer=tokenise_and_stem_text) )\n",
    "                ('countvec_palate', CountVectorizer(\n",
    "                    tokenizer=tokenise_and_stem_text, vocabulary=wordlist) )\n",
    "            ]) ),\n",
    "            ('pipe_finish', Pipeline([\n",
    "                ('get_finish', FunctionTransformer(getfinish)),\n",
    "                #('countvec_finish', CountVectorizer(\n",
    "                #    tokenizer=tokenise_and_stem_text) )\n",
    "                ('countvec_finish', CountVectorizer(\n",
    "                    tokenizer=tokenise_and_stem_text, vocabulary=wordlist) )\n",
    "            ]) ),\n",
    "        ]) ),\n",
    "         \n",
    "        ('tfidf_transform', TfidfTransformer()),\n",
    "        \n",
    "        # Classifiers to try\n",
    "        ('clf_RF', RandomForestClassifier(n_jobs=-1)) #19%\n",
    "        #('clf_kneigh', KNeighborsClassifier()), #6%\n",
    "        #('clf_svc', SVC()), #19%\n",
    "        #('clf_gpc', GaussianProcessClassifier()), #didn't work\n",
    "        #('clf_tree', DecisionTreeClassifier()), #14%\n",
    "        #('clf_MLP', MLPClassifier()), #16%\n",
    "        #('clf_Ada', AdaBoostClassifier()), #4%\n",
    "        #('clf_NBayes', GaussianNB()), #didn't work\n",
    "        #('clf_QuadDA', QuadraticDiscriminantAnalysis()) #didn't work\n",
    "        #('clf_logreg', LogisticRegression()) #17%\n",
    "        #('clf_SGDclf', SGDClassifier()) #15%\n",
    "        ])\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(df[['nose', 'palate', 'finish']], df['brand'])\n",
    "#X_train, X_test = X_train.to_numpy(), X_test.to_numpy()\n",
    "\n",
    "#pipeline.fit(X_train, y_train)\n",
    "\n",
    "#y_pred = pipeline.predict(X_test)\n",
    "\n",
    "#print(classification_report(y_test, y_pred, zero_division=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0abc7f5",
   "metadata": {},
   "source": [
    "As you can see in the above code, I also attempted running things without my custom vocabulary list. Here it was possible to get a small improvement by not using my vocabulary list, but that is effectively cheating since the review will often have the name of the whisky itself in the review, but I was interested to see how much difference it made. Remember that I'm interested in producing the best predictions for users who will be entering nose, palate and finish on an app - it's unlikely they'll be entering the name of the whisky itself, so training it on that data would be pointless even if does improve accuracy.\n",
    "\n",
    "I first had to get a sense of what sort of classifiers might potentially be worth following up on, so I ran RandomForestClassifier, KNeighborsClassifier, SVC, GaussianProcessClassifier, DecisionTreeClassifier, MLPClassifier, AdaBoostClassifier, GaussianNB, QuadraticDiscriminantAnalysis, LogisticRegression, and SGDClassifier with default values to get a sense of which ones would be best to have a go at fine-tuning. SVC, logistic regression and random forest came out well, but SGD and MLP also did fairly well. But because I didn't want a grid search taking forever, I ended up just looking for the best version of SVC, logistic regression and random forest.\n",
    "\n",
    "I was mainly looking at the overall accuracy of the model, and indeed I use scoring='accuracy' for the grid search later. Why accuracy? Well, there wasn't any particular reason to weight type I or type II errors differently. I just wanted an algorithm that maximised the probability of guessing correctly, and that's what accuracy is. And, in all honesty, given that the results weren't that amazing anyway, with everything topping out at 20%, it probably wasn't going to make that much difference anyway."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "fc968591",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "[CV 1/3] END clf=SVC(), clf__C=2, clf__degree=1;, score=0.187 total time= 3.2min\n",
      "[CV 2/3] END clf=SVC(), clf__C=2, clf__degree=1;, score=0.183 total time= 3.2min\n",
      "[CV 3/3] END clf=SVC(), clf__C=2, clf__degree=1;, score=0.188 total time= 3.2min\n",
      "                                         precision    recall  f1-score   support\n",
      "\n",
      "                                   1792       0.50      0.09      0.15        23\n",
      "                              Aberfeldy       0.00      0.00      0.00        13\n",
      "                               Aberlour       0.24      0.25      0.25       120\n",
      "                              Ailsa Bay       0.50      0.25      0.33         4\n",
      "                                 Akashi       0.00      0.00      0.00         5\n",
      "                                Alberta       0.00      0.00      0.00         9\n",
      "                         Allt-A-Bhainne       0.00      0.00      0.00         2\n",
      "                                  Amrut       0.29      0.13      0.18        55\n",
      "                                 AnCnoc       0.20      0.03      0.06        31\n",
      "                            Angels Envy       0.00      0.00      0.00         5\n",
      "                        Angels Envy Rye       0.00      0.00      0.00         4\n",
      "                                 Ardbeg       0.16      0.40      0.23       194\n",
      "                                Ardmore       0.44      0.10      0.16        42\n",
      "                           Ardnamurchan       0.00      0.00      0.00         4\n",
      "                                  Arran       0.07      0.19      0.10        97\n",
      "                           Auchentoshan       0.16      0.10      0.12        50\n",
      "                              Auchroisk       1.00      0.13      0.24        15\n",
      "                               Aultmore       0.00      0.00      0.00        14\n",
      "                                 Bakers       0.00      0.00      0.00         6\n",
      "                               Balblair       0.50      0.05      0.09        20\n",
      "                               Balcones       1.00      0.05      0.10        19\n",
      "                            Ballantines       1.00      0.11      0.20         9\n",
      "                              Ballechin       0.50      0.04      0.08        23\n",
      "                              Balmenach       0.00      0.00      0.00         2\n",
      "                               Balvenie       0.18      0.41      0.25       136\n",
      "                        Barrell Whiskey       0.00      0.00      0.00         3\n",
      "                          Basil Haydens       0.00      0.00      0.00         4\n",
      "                              Ben Nevis       0.71      0.13      0.22        38\n",
      "                               BenRiach       0.30      0.16      0.21        80\n",
      "                              Benrinnes       0.20      0.02      0.04        51\n",
      "                              Benromach       0.50      0.11      0.18        36\n",
      "                           Black Bottle       1.00      0.29      0.44         7\n",
      "                           Black Grouse       0.00      0.00      0.00         5\n",
      "                               Bladnoch       0.33      0.06      0.10        18\n",
      "                            Blair Athol       0.50      0.08      0.13        13\n",
      "                               Blantons       0.62      0.15      0.24        34\n",
      "                                Bookers       0.80      0.43      0.56        28\n",
      "                            Bookers Rye       0.00      0.00      0.00         2\n",
      "                                Bowmore       0.19      0.16      0.17       103\n",
      "                                Braeval       0.00      0.00      0.00         3\n",
      "                                  Brora       0.00      0.00      0.00        12\n",
      "                          Bruichladdich       0.12      0.36      0.18       168\n",
      "                              Buchanans       0.00      0.00      0.00         3\n",
      "                          Buffalo Trace       0.00      0.00      0.00        19\n",
      "                        Bulleit Bourbon       0.00      0.00      0.00        18\n",
      "                            Bulleit Rye       0.00      0.00      0.00         8\n",
      "                           Bunnahabhain       0.13      0.34      0.19       166\n",
      "                              Bushmills       0.00      0.00      0.00        15\n",
      "                            Campbeltown       0.00      0.00      0.00         4\n",
      "                          Canadian Club       1.00      0.18      0.31        11\n",
      "                               Caol Ila       0.16      0.21      0.18       125\n",
      "                            Caperdonich       0.00      0.00      0.00         4\n",
      "                                 Cardhu       1.00      0.08      0.14        13\n",
      "                               Chichibu       0.00      0.00      0.00         5\n",
      "                           Chivas Regal       1.00      0.06      0.11        17\n",
      "                              Clynelish       0.37      0.40      0.38        80\n",
      "                      Colonel EH Taylor       0.25      0.15      0.19        26\n",
      "         Colonel EH Taylor Straight Rye       0.00      0.00      0.00        12\n",
      "                      Compass Box Asyla       0.00      0.00      0.00         4\n",
      "              Compass Box Flaming Heart       0.00      0.00      0.00         7\n",
      "                 Compass Box Great King       0.40      0.11      0.17        19\n",
      "                   Compass Box Hedonism       0.00      0.00      0.00        12\n",
      "                    Compass Box No Name       0.00      0.00      0.00         4\n",
      "                  Compass Box Oak Cross       0.00      0.00      0.00         8\n",
      "               Compass Box Peat Monster       0.20      0.08      0.11        13\n",
      "                 Compass Box Spice Tree       0.00      0.00      0.00         9\n",
      "Compass Box This Is Not a Luxury Whisky       0.00      0.00      0.00         4\n",
      "                              Connemara       0.00      0.00      0.00         5\n",
      "                                 Cooley       0.00      0.00      0.00         2\n",
      "                            Cragganmore       1.00      0.08      0.15        25\n",
      "                          Craigellachie       0.00      0.00      0.00        29\n",
      "                            Crown Royal       1.00      0.11      0.19        19\n",
      "                             Cutty Sark       0.00      0.00      0.00         5\n",
      "                               Daftmill       0.00      0.00      0.00         3\n",
      "                              Dailuaine       0.00      0.00      0.00        18\n",
      "                             Dallas Dhu       0.00      0.00      0.00         3\n",
      "                                Dalmore       0.40      0.10      0.16        40\n",
      "                             Dalwhinnie       1.00      0.07      0.12        15\n",
      "                               Deanston       0.50      0.05      0.09        21\n",
      "                                 Dewars       0.00      0.00      0.00         8\n",
      "                                 Dimple       0.00      0.00      0.00         2\n",
      "                               Dufftown       1.00      0.10      0.18        10\n",
      "                             Eagle Rare       0.40      0.21      0.28        28\n",
      "                               Edradour       0.80      0.10      0.17        41\n",
      "                           Elijah Craig       0.23      0.26      0.24        77\n",
      "                            Elmer T Lee       0.00      0.00      0.00        10\n",
      "                          Evan Williams       0.21      0.24      0.22        38\n",
      "                          Famous Grouse       0.50      0.14      0.22         7\n",
      "                            Fettercairn       1.00      0.08      0.14        13\n",
      "                          Fighting Cock       0.00      0.00      0.00         3\n",
      "                              Finlaggan       1.00      0.14      0.25         7\n",
      "                            Forty Creek       1.00      0.08      0.15        12\n",
      "                             Four Roses       0.15      0.67      0.25       127\n",
      "                          George Dickel       0.50      0.10      0.17        10\n",
      "                         George T Stagg       0.40      0.19      0.26        21\n",
      "                            Glen Breton       1.00      0.33      0.50         3\n",
      "                             Glen Elgin       0.33      0.08      0.13        12\n",
      "                           Glen Garioch       0.40      0.12      0.18        17\n",
      "                             Glen Grant       0.40      0.05      0.09        39\n",
      "                             Glen Keith       1.00      0.08      0.15        12\n",
      "                             Glen Moray       1.00      0.14      0.25        28\n",
      "                               Glen Ord       0.00      0.00      0.00         6\n",
      "                            Glen Scotia       0.00      0.00      0.00        39\n",
      "                              Glen Spey       0.00      0.00      0.00         5\n",
      "                           Glenallachie       1.00      0.05      0.10        20\n",
      "                             Glenburgie       0.00      0.00      0.00        20\n",
      "                              Glencadam       0.00      0.00      0.00         7\n",
      "                            Glendronach       0.16      0.45      0.23       129\n",
      "                             Glendullan       0.00      0.00      0.00         7\n",
      "                            Glenfarclas       0.15      0.20      0.17       107\n",
      "                            Glenfiddich       0.22      0.29      0.25       105\n",
      "                          Glenglassaugh       0.00      0.00      0.00         6\n",
      "                              Glengoyne       0.20      0.02      0.04        44\n",
      "                            Glenkinchie       1.00      0.07      0.13        14\n",
      "                              Glenlivet       0.10      0.28      0.15       106\n",
      "                             Glenlossie       0.00      0.00      0.00        11\n",
      "                           Glenmorangie       0.13      0.26      0.18       121\n",
      "                             Glenrothes       0.50      0.07      0.12        30\n",
      "                           Glentauchers       0.00      0.00      0.00        20\n",
      "                             Glenturret       0.00      0.00      0.00        13\n",
      "                      Gooderham & Worts       0.00      0.00      0.00         5\n",
      "                             Green Spot       0.50      0.11      0.18         9\n",
      "                                Hakushu       0.67      0.20      0.31        10\n",
      "                              Hazelburn       0.00      0.00      0.00        16\n",
      "                              Heartwood       0.80      0.36      0.50        11\n",
      "                            Heaven Hill       0.00      0.00      0.00        11\n",
      "                          Hellyers Road       1.00      0.17      0.29         6\n",
      "                          Henry McKenna       1.00      0.06      0.11        18\n",
      "                                 Hibiki       1.00      0.08      0.15        12\n",
      "                              High West       0.33      0.06      0.10        17\n",
      "                          Highland Park       0.19      0.27      0.22       151\n",
      "                                Ichiros       0.00      0.00      0.00         2\n",
      "                                 Ileach       0.00      0.00      0.00         6\n",
      "                               Imperial       1.00      0.14      0.25         7\n",
      "                              Inchgower       1.00      0.11      0.20         9\n",
      "                             Inchmurrin       0.00      0.00      0.00         6\n",
      "                               JP Wiser       0.29      0.11      0.16        18\n",
      "                           Jack Daniels       0.50      0.29      0.37        24\n",
      "                         James E Pepper       0.00      0.00      0.00         1\n",
      "                                Jameson       0.60      0.20      0.30        15\n",
      "                             Jeffersons       0.00      0.00      0.00        12\n",
      "                               Jim Beam       0.19      0.33      0.24        15\n",
      "                         Johnnie Walker       0.27      0.41      0.33       100\n",
      "                                   Jura       0.50      0.04      0.08        47\n",
      "                              Karuizawa       0.00      0.00      0.00         5\n",
      "                                Kavalan       0.73      0.22      0.33        37\n",
      "                       Kentucky Owl Rye       0.00      0.00      0.00         4\n",
      "                              Kilchoman       0.29      0.13      0.18        98\n",
      "                              Kilkerran       1.00      0.22      0.36        36\n",
      "                       Knappogue Castle       0.00      0.00      0.00         1\n",
      "                             Knob Creek       0.31      0.24      0.27        37\n",
      "                         Knob Creek Rye       0.00      0.00      0.00         6\n",
      "                              Knockando       1.00      0.50      0.67         2\n",
      "                              Lagavulin       0.40      0.22      0.28       101\n",
      "                              Laphroaig       0.31      0.56      0.40       274\n",
      "                                Larceny       1.00      0.11      0.20         9\n",
      "                                   Lark       0.00      0.00      0.00         3\n",
      "                                 Ledaig       0.55      0.20      0.30        79\n",
      "                               Linkwood       0.29      0.09      0.14        22\n",
      "                                Lismore       0.00      0.00      0.00         2\n",
      "                             Littlemill       0.00      0.00      0.00        12\n",
      "                            Loch Lomond       0.00      0.00      0.00         7\n",
      "                               Longmorn       0.00      0.00      0.00        27\n",
      "                                Longrow       0.58      0.24      0.34        58\n",
      "                              Lot No 40       1.00      0.25      0.40         8\n",
      "                               Macallan       0.14      0.25      0.18       122\n",
      "                  Mackinlays Shackleton       0.00      0.00      0.00         9\n",
      "                            Makers Mark       0.31      0.31      0.31        35\n",
      "                            Mannochmore       0.00      0.00      0.00         8\n",
      "                              McCarthys       0.00      0.00      0.00         3\n",
      "                            Mellow Corn       0.00      0.00      0.00         7\n",
      "                               Michters       0.00      0.00      0.00         8\n",
      "                           Michters Rye       0.33      0.11      0.17         9\n",
      "                               Midleton       0.00      0.00      0.00         4\n",
      "                             Miltonduff       0.00      0.00      0.00         3\n",
      "                        Monkey Shoulder       1.00      0.08      0.15        12\n",
      "                               Mortlach       0.42      0.10      0.16        52\n",
      "                               New Riff       1.00      0.33      0.50         6\n",
      "                                  Nikka       0.73      0.17      0.27        48\n",
      "                             Noahs Mill       0.00      0.00      0.00         5\n",
      "                                   Oban       0.33      0.03      0.05        36\n",
      "                               Octomore       0.44      0.25      0.32        75\n",
      "                            Old Charter       0.00      0.00      0.00         5\n",
      "                               Old Crow       0.00      0.00      0.00         4\n",
      "                               Old Ezra       0.00      0.00      0.00         5\n",
      "                         Old Fitzgerald       1.00      0.11      0.20         9\n",
      "                           Old Forester       0.22      0.07      0.11        28\n",
      "                          Old Grand Dad       1.00      0.16      0.28        25\n",
      "                           Old Overholt       1.00      0.20      0.33         5\n",
      "                              Old Perth       0.00      0.00      0.00         5\n",
      "                           Old Pulteney       0.31      0.20      0.25        44\n",
      "                     Old Rip Van Winkle       0.00      0.00      0.00        13\n",
      "                     Old Weller Antique       0.14      0.09      0.11        23\n",
      "                                   Omar       0.50      0.17      0.25         6\n",
      "                          Orphan Barrel       1.00      0.14      0.25        14\n",
      "                              Paul John       1.00      0.11      0.20         9\n",
      "                              Pigs Nose       0.00      0.00      0.00         5\n",
      "                             Pike Creek       0.00      0.00      0.00         3\n",
      "                             Pikesville       0.00      0.00      0.00         9\n",
      "                             Pittyvaich       0.00      0.00      0.00         1\n",
      "                            Port Askaig       0.00      0.00      0.00         6\n",
      "                         Port Charlotte       0.37      0.08      0.13        88\n",
      "                             Port Ellen       1.00      0.06      0.11        18\n",
      "                                 Powers       0.00      0.00      0.00         2\n",
      "                             Rebel Yell       1.00      0.14      0.25         7\n",
      "                              Redbreast       1.00      0.08      0.15        24\n",
      "                             Redemption       0.00      0.00      0.00         6\n",
      "                            Rittenhouse       0.00      0.00      0.00        15\n",
      "                        Rock Hill Farms       0.00      0.00      0.00         4\n",
      "                               Rosebank       0.00      0.00      0.00         6\n",
      "                           Rowans Creek       0.00      0.00      0.00         4\n",
      "                          Royal Brackla       0.00      0.00      0.00         4\n",
      "                        Royal Lochnagar       1.00      0.43      0.60         7\n",
      "                       Russells Reserve       0.62      0.24      0.34        34\n",
      "                   Russells Reserve Rye       1.00      0.11      0.20         9\n",
      "                                Sazerac       1.00      0.05      0.09        22\n",
      "                                  Scapa       0.00      0.00      0.00        16\n",
      "                              Sheep Dip       0.00      0.00      0.00         6\n",
      "                          Smooth Ambler       1.00      0.17      0.29        12\n",
      "                               Speyburn       0.00      0.00      0.00         5\n",
      "                             Springbank       0.27      0.35      0.31       143\n",
      "                               Stagg Jr       0.67      0.15      0.25        26\n",
      "                           Still Waters       1.00      0.17      0.29         6\n",
      "                             Stranahans       0.00      0.00      0.00        10\n",
      "                             Strathisla       0.00      0.00      0.00         6\n",
      "                             Strathmill       0.50      0.11      0.18         9\n",
      "                             Stronachie       0.00      0.00      0.00         5\n",
      "                         Sullivans Cove       1.00      0.20      0.33         5\n",
      "                               Talisker       0.38      0.32      0.35       116\n",
      "                                 Tamdhu       0.00      0.00      0.00        23\n",
      "                             Tamnavulin       0.00      0.00      0.00         7\n",
      "                               Teachers       0.00      0.00      0.00         2\n",
      "                                Teeling       1.00      0.29      0.44        14\n",
      "                         Thomas H Handy       0.60      0.14      0.23        21\n",
      "                              Tobermory       0.00      0.00      0.00        28\n",
      "                                Tomatin       0.89      0.14      0.25        56\n",
      "                              Tomintoul       1.00      0.06      0.11        17\n",
      "                                Tormore       0.00      0.00      0.00         6\n",
      "                                Toronto       1.00      0.50      0.67         2\n",
      "                          Tullamore Dew       0.00      0.00      0.00         3\n",
      "                           Tullibardine       0.00      0.00      0.00         9\n",
      "                             Tyrconnell       1.00      0.33      0.50         6\n",
      "                             Van Winkle       0.00      0.00      0.00        21\n",
      "                        Very Old Barton       0.00      0.00      0.00         6\n",
      "                              WL Weller       0.22      0.07      0.11        27\n",
      "                               Westland       0.50      0.05      0.08        22\n",
      "                             WhistlePig       0.22      0.24      0.23        33\n",
      "                            Wild Turkey       0.22      0.37      0.28        81\n",
      "                    Wild Turkey 101 Rye       0.00      0.00      0.00         7\n",
      "                                Willett       0.67      0.10      0.17        20\n",
      "              Willett Family Estate Rye       0.11      0.08      0.09        26\n",
      "                       Woodford Reserve       0.75      0.17      0.27        18\n",
      "                          Writers Tears       0.00      0.00      0.00         4\n",
      "                               Yamazaki       0.50      0.08      0.13        13\n",
      "                            Yellow Spot       0.00      0.00      0.00         2\n",
      "\n",
      "                               accuracy                           0.21      6739\n",
      "                              macro avg       0.33      0.10      0.13      6739\n",
      "                           weighted avg       0.32      0.21      0.19      6739\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Helper functions for getting the right column of data to use in the pipeline:\n",
    "def getnose(array):\n",
    "    return array[:,0]\n",
    "    \n",
    "def getpalate(array):\n",
    "    return array[:,1]\n",
    "\n",
    "def getfinish(array):\n",
    "    return array[:,2]\n",
    "\n",
    "# Pipeline to use\n",
    "pipeline = Pipeline([\n",
    "        \n",
    "        ('union', FeatureUnion([\n",
    "            \n",
    "            ('pipe_nose', Pipeline([\n",
    "                ('get_nose', FunctionTransformer(getnose)),\n",
    "                ('countvec_nose', CountVectorizer(\n",
    "                    tokenizer=tokenise_and_stem_text, vocabulary=wordlist) )\n",
    "            ]) ),\n",
    "            ('pipe_palate', Pipeline([\n",
    "                ('get_palate', FunctionTransformer(getpalate)),\n",
    "                ('countvec_palate', CountVectorizer(\n",
    "                    tokenizer=tokenise_and_stem_text, vocabulary=wordlist) )\n",
    "            ]) ),\n",
    "            ('pipe_finish', Pipeline([\n",
    "                ('get_finish', FunctionTransformer(getfinish)),\n",
    "                ('countvec_finish', CountVectorizer(\n",
    "                    tokenizer=tokenise_and_stem_text, vocabulary=wordlist) )\n",
    "            ]) ),\n",
    "        ]) ),\n",
    "         \n",
    "        ('tfidf_transform', TfidfTransformer()),\n",
    "        \n",
    "        ('clf', RandomForestClassifier(n_jobs=-1))\n",
    "        ])\n",
    "\n",
    "# Grid for grid search \n",
    "param_grid = [\n",
    "        {'clf' : [LogisticRegression()],\n",
    "         'clf__penalty' : ['l2'],\n",
    "        'clf__C' : [1, 2, 4, 8],\n",
    "        'clf__solver' : ['liblinear']},\n",
    "        {'clf' : [RandomForestClassifier(n_jobs=-1)],\n",
    "        'clf__bootstrap': [False],\n",
    "        'clf__max_depth': [200, 500],\n",
    "        'clf__max_features': ['sqrt'],\n",
    "        'clf__n_estimators': [1000, 2000]},\n",
    "        {'clf' : [SVC()],\n",
    "         'clf__degree' : [1, 2],\n",
    "         'clf__C' : [2, 2.5, 3, 3.5, 4]}\n",
    "    ]\n",
    "\n",
    "\n",
    "# First of all, do the whole set of whiskies\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    df[['nose', 'palate', 'finish']], df['brand'])\n",
    "X_train, X_test = X_train.to_numpy(), X_test.to_numpy()\n",
    "\n",
    "grid_search_all_whiskies = GridSearchCV(estimator = pipeline, cv=3, \n",
    "                   param_grid = param_grid, scoring='accuracy', verbose=3)\n",
    "\n",
    "grid_search_all_whiskies.fit(X_train, y_train)\n",
    "\n",
    "y_pred = grid_search_all_whiskies.predict(X_test)\n",
    "\n",
    "print(classification_report(y_test, y_pred, zero_division=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bbb7476",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save to pickle file. Commented out because this will not work in the app. The pickle file\n",
    "# will not work when it has modules with '__main__' in them. Instead, you need to create the\n",
    "# pickle file by running get_whisky_clf.py. \n",
    "\n",
    "#with open('./models/whisky_classifier.pkl', \"wb\") as f:\n",
    "#    joblib.dump(grid_search_all_whiskies, f, compress='zlib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "543f0eff",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "\"['param_clf__max_depth', 'param_clf__n_estimators'] not in index\"",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Input \u001b[1;32mIn [23]\u001b[0m, in \u001b[0;36m<cell line: 3>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# Show the results of the gridsearch\u001b[39;00m\n\u001b[0;32m      2\u001b[0m gridsearchresults \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mDataFrame(grid_search_all_whiskies\u001b[38;5;241m.\u001b[39mcv_results_)\n\u001b[1;32m----> 3\u001b[0m \u001b[43mgridsearchresults\u001b[49m\u001b[43m[\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mparam_clf\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mparam_clf__C\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mparam_clf__max_depth\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mparam_clf__n_estimators\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mparam_clf__degree\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mmean_test_score\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m]\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\whisky_env\\lib\\site-packages\\pandas\\core\\frame.py:3511\u001b[0m, in \u001b[0;36mDataFrame.__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   3509\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m is_iterator(key):\n\u001b[0;32m   3510\u001b[0m         key \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(key)\n\u001b[1;32m-> 3511\u001b[0m     indexer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcolumns\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_indexer_strict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcolumns\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m[\u001b[38;5;241m1\u001b[39m]\n\u001b[0;32m   3513\u001b[0m \u001b[38;5;66;03m# take() does not accept boolean indexers\u001b[39;00m\n\u001b[0;32m   3514\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mgetattr\u001b[39m(indexer, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdtype\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m) \u001b[38;5;241m==\u001b[39m \u001b[38;5;28mbool\u001b[39m:\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\whisky_env\\lib\\site-packages\\pandas\\core\\indexes\\base.py:5782\u001b[0m, in \u001b[0;36mIndex._get_indexer_strict\u001b[1;34m(self, key, axis_name)\u001b[0m\n\u001b[0;32m   5779\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m   5780\u001b[0m     keyarr, indexer, new_indexer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reindex_non_unique(keyarr)\n\u001b[1;32m-> 5782\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_raise_if_missing\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkeyarr\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maxis_name\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   5784\u001b[0m keyarr \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtake(indexer)\n\u001b[0;32m   5785\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(key, Index):\n\u001b[0;32m   5786\u001b[0m     \u001b[38;5;66;03m# GH 42790 - Preserve name from an Index\u001b[39;00m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\whisky_env\\lib\\site-packages\\pandas\\core\\indexes\\base.py:5845\u001b[0m, in \u001b[0;36mIndex._raise_if_missing\u001b[1;34m(self, key, indexer, axis_name)\u001b[0m\n\u001b[0;32m   5842\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNone of [\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mkey\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m] are in the [\u001b[39m\u001b[38;5;132;01m{\u001b[39;00maxis_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m]\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m   5844\u001b[0m not_found \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(ensure_index(key)[missing_mask\u001b[38;5;241m.\u001b[39mnonzero()[\u001b[38;5;241m0\u001b[39m]]\u001b[38;5;241m.\u001b[39munique())\n\u001b[1;32m-> 5845\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mnot_found\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m not in index\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[1;31mKeyError\u001b[0m: \"['param_clf__max_depth', 'param_clf__n_estimators'] not in index\""
     ]
    }
   ],
   "source": [
    "# Show the results of the gridsearch\n",
    "gridsearchresults = pd.DataFrame(grid_search_all_whiskies.cv_results_)\n",
    "gridsearchresults[['param_clf', 'param_clf__C', 'param_clf__max_depth', 'param_clf__n_estimators', 'param_clf__degree', 'mean_test_score']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a02148e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Testing out prediction code\n",
    "test = np.array([['','',''],['fragrant, vanilla, melon, grass', 'grass, sherry, oaky', 'calm, balanced']])\n",
    "\n",
    "grid_search_all_whiskies.predict(test)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5eb38cd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Then do just the scotch single malts\n",
    "X2_train, X2_test, y2_train, y2_test = train_test_split(\n",
    "    df_sms[['nose', 'palate', 'finish']], df_sms['brand'])\n",
    "X2_train, X2_test = X2_train.to_numpy(), X2_test.to_numpy()\n",
    "\n",
    "grid_search_single_malts = GridSearchCV(estimator = pipeline, cv=3, \n",
    "                    param_grid = param_grid, scoring='accuracy', verbose=3)\n",
    "\n",
    "grid_search_single_malts.fit(X2_train, y2_train)\n",
    "\n",
    "y2_pred = grid_search_single_malts.predict(X2_test)\n",
    "\n",
    "print(classification_report(y2_test, y2_pred, zero_division=0))\n",
    "\n",
    "# Save to pickle file\n",
    "#with open('./models/malt_classifier.pkl', \"wb\") as f:\n",
    "#    joblib.dump(grid_search_single_malts, f, compress='zlib')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85915522",
   "metadata": {},
   "source": [
    "In the end, I decided not to proceed with the 'Just single malts' version. It did slightly better, with 23% accuracy as opposed to 21%, but not enough to justify the exclusion of so many interesting whiskies that make the app a truly international affair.\n",
    "\n",
    "As for the score itself, I doubt anyone is going to be blown away by an accuracy of 21%, but informal feedback from people who know their whisky is fairly positive. Even when the guesses are wrong, they make sense given the tasting notes people put in. The model is producing a multi-dimensional matrix where the Euclidean distance does seem to correspond more or less to one's intuitive sense of which whiskies are similar to each other, which is a good sign. "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}