KWS_seminar.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "colab": {
   "name": "KWS_seminar.ipynb",
   "provenance": [],
   "collapsed_sections": []
  },
  "accelerator": "GPU"
 },
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VUDrGrgWqUYY"
   },
   "source": [
    "This notebook is dedicated to Keyword Spotting (KWS)"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "jFeWCV69HPPW",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1636485102938,
     "user_tz": -180,
     "elapsed": 375,
     "user": {
      "displayName": "Илья Седунов",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GikFS5GQ7LVkJtcHQDIxRfWE2Ddr8NPueLJC7XEdw=s64",
      "userId": "17123457789087244305"
     }
    }
   },
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ],
   "execution_count": 10,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "nCTpqrCIjiO1"
   },
   "source": [
    "### Download file for easier downloading and dataset creation\n",
    "!wget https://gist.githubusercontent.com/Kirili4ik/6ac5c745ff8dad094e9c464c08f66f3e/raw/63daacc17f52a7d90f7f4166a3f5deef62b165db/dataset_utils.py\n",
    "#!pip install wandb\n",
    "!pip install easydict\n",
    "!pip install --no-deps torchaudio==0.9.0"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "DwVLim8uGL2w",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1636486361226,
     "user_tz": -180,
     "elapsed": 385,
     "user": {
      "displayName": "Илья Седунов",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GikFS5GQ7LVkJtcHQDIxRfWE2Ddr8NPueLJC7XEdw=s64",
      "userId": "17123457789087244305"
     }
    }
   },
   "source": [
    "from utils.utils import *\n",
    "set_seed(21)"
   ],
   "execution_count": 32,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "812GwLfqqUYf"
   },
   "source": [
    "### Task"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "i1DuQIyRqUYf"
   },
   "source": [
    "In this notebook we will implement a model for finding a keyword in a stream.\n",
    "\n",
    "We will implement the version with CRNN because it is easy and improves the model. \n",
    "(from https://www.dropbox.com/s/22ah2ba7dug6pzw/KWS_Attention.pdf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "oaN4kVAwqUYg"
   },
   "source": [
    "### Configuration"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "CLPSxgf-qUYg",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1636485180439,
     "user_tz": -180,
     "elapsed": 418,
     "user": {
      "displayName": "Илья Седунов",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GikFS5GQ7LVkJtcHQDIxRfWE2Ddr8NPueLJC7XEdw=s64",
      "userId": "17123457789087244305"
     }
    },
    "outputId": "3acbd5c5-2469-4cd4-e293-fb42d2bd2d64"
   },
   "source": [
    "key_word = 'sheila'   # We will use 1 key word -- 'sheila'\n",
    "\n",
    "config = {\n",
    "    'key_word'      : key_word,\n",
    "    'batch_size'    : 256,\n",
    "    'learning_rate' : 3e-4,\n",
    "    'weight_decay'  : 1e-5,\n",
    "    'num_epochs'    : 35,\n",
    "    'n_mels'        : 40,         # number of mels for melspectrogram\n",
    "    'kernel_size'   : (20, 5),    # size of kernel for convolution layer in CRNN\n",
    "    'stride'        : (8, 2),     # size of stride for convolution layer in CRNN\n",
    "    'hidden_size'   : 128,        # size of hidden representation in GRU\n",
    "    'gru_num_layers': 2,          # number of GRU layers in CRNN\n",
    "    'gru_num_dirs'  : 2,          # number of directions in GRU (2 if bidirectional)\n",
    "    'num_classes'   : 2,          # number of classes (2 for \"no word\" or \"sheila is in audio\")\n",
    "    'sample_rate'   : 16000,\n",
    "    'device'        : torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "}\n",
    "\n",
    "config = make_config(key_word, config)\n",
    "print(f\"keyword: '{config.key_word}'\\ndevice: {config.device}\")"
   ],
   "execution_count": 18,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "keyword: 'sheila'\n",
      "device: cuda:0\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "doBqqRX5qUYb"
   },
   "source": [
    "#### Download, generate lables & create Datasets:"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "vGiCUehYivKw",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 250
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1636485750596,
     "user_tz": -180,
     "elapsed": 459370,
     "user": {
      "displayName": "Илья Седунов",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GikFS5GQ7LVkJtcHQDIxRfWE2Ddr8NPueLJC7XEdw=s64",
      "userId": "17123457789087244305"
     }
    },
    "outputId": "4db98599-d14f-4a9d-f1b2-60fb141125bd"
   },
   "source": [
    "from dataset_utils import DatasetDownloader\n",
    "\n",
    "dataset_downloader = DatasetDownloader(key_word)\n",
    "labeled_data, _ = dataset_downloader.generate_labeled_data()\n",
    "\n",
    "labeled_data.sample(3)"
   ],
   "execution_count": 19,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Downloading data...\n",
      "Ready!\n",
      "Classes: bed, bird, cat, dog, down, eight, five, four, go, happy, house, left, marvin, nine, no, off, on, one, right, seven, sheila, six, stop, three, tree, two, up, wow, yes, zero\n",
      "Creating labeled dataframe:\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "100%|██████████| 31/31 [06:28<00:00, 12.52s/it]\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>word</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>63125</th>\n",
       "      <td>speech_commands/marvin/c8db14a8_nohash_1.wav</td>\n",
       "      <td>marvin</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5266</th>\n",
       "      <td>speech_commands/five/1a9afd33_nohash_0.wav</td>\n",
       "      <td>five</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61600</th>\n",
       "      <td>speech_commands/wow/dc75148d_nohash_0.wav</td>\n",
       "      <td>wow</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               name    word label\n",
       "63125  speech_commands/marvin/c8db14a8_nohash_1.wav  marvin     0\n",
       "5266     speech_commands/five/1a9afd33_nohash_0.wav    five     0\n",
       "61600     speech_commands/wow/dc75148d_nohash_0.wav     wow     0"
      ]
     },
     "metadata": {},
     "execution_count": 19
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "PDPLht5fqUYe",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1636486078646,
     "user_tz": -180,
     "elapsed": 436,
     "user": {
      "displayName": "Илья Седунов",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GikFS5GQ7LVkJtcHQDIxRfWE2Ddr8NPueLJC7XEdw=s64",
      "userId": "17123457789087244305"
     }
    },
    "outputId": "66a3d66b-5362-4b96-90a5-bb35c1588c3b"
   },
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from augmentations.augs_creation import AugsCreation\n",
    "\n",
    "# create 2 dataframes for train/val so we can use augmentations only for train\n",
    "train_df, val_df = train_test_split(labeled_data, test_size=0.2, stratify=labeled_data['label'],  random_state=21)\n",
    "train_df, val_df = train_df.reset_index(drop=True), val_df.reset_index(drop=True)\n",
    "\n",
    "\n",
    "from dataset_utils import TrainDataset\n",
    "\n",
    "# Sample is a dict of utt, word and label\n",
    "transform_tr = AugsCreation()\n",
    "train_set = TrainDataset(df=train_df, kw=config.key_word, transform=transform_tr)\n",
    "val_set   = TrainDataset(df=val_df,   kw=config.key_word)\n",
    "\n",
    "print('all train + val samples:', len(train_set)+len(val_set))"
   ],
   "execution_count": 24,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "all train + val samples: 64721\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2vbPDqd6qUYj"
   },
   "source": [
    "#### Sampler for oversampling:"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "UM8gLmHeqUYj",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1636486137284,
     "user_tz": -180,
     "elapsed": 425,
     "user": {
      "displayName": "Илья Седунов",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GikFS5GQ7LVkJtcHQDIxRfWE2Ddr8NPueLJC7XEdw=s64",
      "userId": "17123457789087244305"
     }
    }
   },
   "source": [
    "train_sampler = get_sampler(train_set.df['label'].values)\n",
    "val_sampler   = get_sampler(val_set.df['label'].values)"
   ],
   "execution_count": 27,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "e8G9xPRVqUYk"
   },
   "source": [
    "###  Dataloaders"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "6wGBMcQiqUYk",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1636486167929,
     "user_tz": -180,
     "elapsed": 382,
     "user": {
      "displayName": "Илья Седунов",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GikFS5GQ7LVkJtcHQDIxRfWE2Ddr8NPueLJC7XEdw=s64",
      "userId": "17123457789087244305"
     }
    }
   },
   "source": [
    "# Here we are obliged to use shuffle=False because of our sampler with randomness inside.\n",
    "\n",
    "train_loader = DataLoader(train_set, batch_size=config.batch_size,\n",
    "                          shuffle=False, collate_fn=batch_data, \n",
    "                          sampler=train_sampler,\n",
    "                          num_workers=2, pin_memory=True)\n",
    "\n",
    "val_loader = DataLoader(val_set, batch_size=config.batch_size,\n",
    "                        shuffle=False, collate_fn=batch_data, \n",
    "                        sampler=val_sampler,\n",
    "                        num_workers=2, pin_memory=True)"
   ],
   "execution_count": 29,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kTlsn6cpqUYk"
   },
   "source": [
    "### Creating MelSpecs on GPU for speeeed: "
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "Pqkz4_gn8BiF",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1636486291119,
     "user_tz": -180,
     "elapsed": 14982,
     "user": {
      "displayName": "Илья Седунов",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GikFS5GQ7LVkJtcHQDIxRfWE2Ddr8NPueLJC7XEdw=s64",
      "userId": "17123457789087244305"
     }
    }
   },
   "source": [
    "from preprocessing.log_mel_spec import LogMelspec\n",
    "\n",
    "melspec_train = LogMelspec(is_train=True, config=config)\n",
    "melspec_val = LogMelspec(is_train=False, config=config)"
   ],
   "execution_count": 31,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CcEP5cEZqUYl"
   },
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "scrolled": true,
    "id": "P3W-MOJLqUYm",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1636486538161,
     "user_tz": -180,
     "elapsed": 4,
     "user": {
      "displayName": "Илья Седунов",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GikFS5GQ7LVkJtcHQDIxRfWE2Ddr8NPueLJC7XEdw=s64",
      "userId": "17123457789087244305"
     }
    },
    "outputId": "e33f2b71-a597-4e46-bcd1-f8cbd6cfd561"
   },
   "source": [
    "from model.model import *\n",
    "\n",
    "CRNN_model = CRNN(config)\n",
    "\n",
    "attn_layer = AttnMech(config)\n",
    "\n",
    "full_model = FullModel(config, CRNN_model, attn_layer)\n",
    "\n",
    "full_model = full_model.to(config.device)\n",
    "\n",
    "print(full_model)"
   ],
   "execution_count": 33,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "FullModel(\n",
      "  (CRNN_model): CRNN(\n",
      "    (sepconv): Sequential(\n",
      "      (0): Conv1d(40, 40, kernel_size=(5,), stride=(2,), groups=40)\n",
      "      (1): Conv1d(40, 128, kernel_size=(1,), stride=(8,), groups=2)\n",
      "    )\n",
      "    (gru): GRU(128, 128, num_layers=2, dropout=0.1, bidirectional=True)\n",
      "  )\n",
      "  (attn_layer): AttnMech(\n",
      "    (Wx_b): Linear(in_features=256, out_features=256, bias=True)\n",
      "    (Vt): Linear(in_features=256, out_features=1, bias=False)\n",
      "  )\n",
      "  (U): Linear(in_features=256, out_features=2, bias=False)\n",
      ")\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "ev287I6SqUYn",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1636486547077,
     "user_tz": -180,
     "elapsed": 806,
     "user": {
      "displayName": "Илья Седунов",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GikFS5GQ7LVkJtcHQDIxRfWE2Ddr8NPueLJC7XEdw=s64",
      "userId": "17123457789087244305"
     }
    }
   },
   "source": [
    "opt = torch.optim.Adam(full_model.parameters(), lr=config.learning_rate, weight_decay=config.weight_decay)"
   ],
   "execution_count": 34,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "scrolled": true,
    "id": "32oooz4lqUYo"
   },
   "source": [
    "### TRAIN\n",
    "from train_utils.utils import *\n",
    "\n",
    "for n in range(config.num_epochs):\n",
    "    \n",
    "    train_epoch(full_model, opt, train_loader, melspec_train,  \n",
    "                config.gru_num_layers, config.gru_num_dirs,\n",
    "                config.hidden_size, config.device)           \n",
    "        \n",
    "    validation(full_model, val_loader, melspec_val,\n",
    "               config.gru_num_layers, config.gru_num_dirs,\n",
    "               config.hidden_size, config.device)\n",
    "\n",
    "    print('END OF EPOCH', n)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "aWaiioNzSQus"
   },
   "source": [
    ""
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "SJee_D6LqUYo"
   },
   "source": [
    "torch.save({\n",
    "    'model_state_dict': full_model.state_dict(),\n",
    "}, 'base_35ep')"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "NUofpIZL3WRJ",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1636487204530,
     "user_tz": -180,
     "elapsed": 33229,
     "user": {
      "displayName": "Илья Седунов",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GikFS5GQ7LVkJtcHQDIxRfWE2Ddr8NPueLJC7XEdw=s64",
      "userId": "17123457789087244305"
     }
    },
    "outputId": "4440d057-ceb8-4d84-fe52-bbaf8b7cfd78"
   },
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/gdrive')"
   ],
   "execution_count": 36,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Mounted at /gdrive\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9p4uZZssPqCy",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1636487347102,
     "user_tz": -180,
     "elapsed": 548,
     "user": {
      "displayName": "Илья Седунов",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GikFS5GQ7LVkJtcHQDIxRfWE2Ddr8NPueLJC7XEdw=s64",
      "userId": "17123457789087244305"
     }
    },
    "outputId": "aed308c6-4765-4e6e-9ba3-bee16c2d549d"
   },
   "source": [
    "!ls /gdrive/MyDrive/DLA/"
   ],
   "execution_count": 39,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "KWS_seminar.ipynb  Untitled0.ipynb\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "2DiKxgtDP2fA",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1636487557846,
     "user_tz": -180,
     "elapsed": 431,
     "user": {
      "displayName": "Илья Седунов",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GikFS5GQ7LVkJtcHQDIxRfWE2Ddr8NPueLJC7XEdw=s64",
      "userId": "17123457789087244305"
     }
    }
   },
   "source": [
    "!cp -r augmentations dataset_utils.py preprocessing model train_utils utils /gdrive/MyDrive/DLA/"
   ],
   "execution_count": 43,
   "outputs": []
  }
 ]
}