task10_bert_squad.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BERT для вопросно-ответных систем"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Если Вы запускаете ноутбук на colab или kaggle,\n",
    "# выполните следующие строчки, чтобы подгрузить библиотеку dlnlputils:\n",
    "\n",
    "# !git clone https://github.com/Samsung-IT-Academy/stepik-dl-nlp.git && pip install -r stepik-dl-nlp/requirements.txt\n",
    "# import sys; sys.path.append('./stepik-dl-nlp')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Скачайте датасет (SQuAD) [отсюда](https://rajpurkar.github.io/SQuAD-explorer/). Для выполенения семинара Вам понадобятся файлы `train-v2.0.json` и `dev-v2.0.json`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Склонируйте репозиторий https://github.com/huggingface/transformers (воспользуйтесь скриптом `clone_pytorch_transformers.sh`) и положите путь до папки `examples` в переменную `PATH_TO_EXAMPLES`. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH_TO_TRANSFORMERS_REPO = '../transformers/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['PATH_TO_TRANSFORMER_REPO'] = PATH_TO_TRANSFORMERS_REPO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! bash clone_pytorch_transformers.sh $PATH_TO_TRANSFORMERS_REPO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "PATH_TO_EXAMPLES = os.path.join(PATH_TO_TRANSFORMERS_REPO, 'examples')\n",
    "sys.path.append(PATH_TO_EXAMPLES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import tqdm\n",
    "import json\n",
    "\n",
    "from utils_squad import (read_squad_examples, convert_examples_to_features,\n",
    "                         RawResult, write_predictions,\n",
    "                         RawResultExtended, write_predictions_extended)\n",
    "\n",
    "from run_squad import train, load_and_cache_examples\n",
    "\n",
    "from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, TensorDataset)\n",
    "\n",
    "from transformers import (WEIGHTS_NAME, BertConfig, XLNetConfig, XLMConfig,\n",
    "                          BertForQuestionAnswering, BertTokenizer)\n",
    "\n",
    "from utils_squad_evaluate import EVAL_OPTS, main as evaluate_on_squad\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True, do_basic_tokenize=True)\n",
    "model = BertForQuestionAnswering.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# если Вы не хотите запускать файн-тюнинг, пропустите блок \"Дообучение\",\n",
    "# подгрузите веса уже дообученной модели и переходите к блоку \"Оценка качества\"\n",
    "\n",
    "# скачайте веса с Google-диска и положите их в папку models\n",
    "# https://drive.google.com/drive/folders/1-DR30q7MF-gZ51TDx596dAOhgh-uOAPj?usp=sharing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "    model.cuda()\n",
    "    model.load_state_dict(torch.load('models/bert_squad_1epoch.pt')) # если у вас есть GPU\n",
    "else:\n",
    "    model.load_state_dict(torch.load('models/bert_squad_1epoch.pt', map_location=device)) # если GPU нет"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Дообучение"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install dataclasses\n",
    "from dataclasses import dataclass\n",
    "\n",
    "@dataclass\n",
    "class TRAIN_OPTS:\n",
    "    train_file : str = 'train-v2.0.json'    # SQuAD json-файл для обучения\n",
    "    predict_file : str = 'dev-v2.0.json'    # SQuAD json-файл для тестирования\n",
    "    model_type : str = 'bert'               # тип модели (может быть  'bert', 'xlnet', 'xlm', 'distilbert')\n",
    "    model_name_or_path : str = 'bert-base-uncased' # путь до предобученной модели или название модели из ALL_MODELS\n",
    "    output_dir : str = '/tmp' # путь до директории, где будут храниться чекпоинты и предсказания модели\n",
    "    device : str = 'cuda' # cuda или cpu\n",
    "    n_gpu : int = 1 # количество gpu для обучения\n",
    "    cache_dir : str = '' # где хранить предобученные модели, загруженные с s3\n",
    "        \n",
    "    # Если true, то в датасет будут включены вопросы, на которые нет ответов.\n",
    "    version_2_with_negative : bool = True\n",
    "    # Если (null_score - best_non_null) больше, чем порог, предсказывать null.\n",
    "    null_score_diff_threshold : float = 0.0\n",
    "    # Максимальная длина входной последовательности после WordPiece токенизации. Sequences \n",
    "    # Последовательности длиннее будут укорочены, для более коротких последовательностей будет использован паддинг\n",
    "    max_seq_length : int = 384\n",
    "    # Сколько stride использовать при делении длинного документа на чанки\n",
    "    doc_stride : int = 128\n",
    "    # Максимальное количество токенов в вопросе. Более длинные вопросы будут укорочены до этой длины\n",
    "    max_query_length : int = 128 #\n",
    "        \n",
    "    do_train : bool = True\n",
    "    do_eval : bool = True\n",
    "        \n",
    "    # Запускать ли evaluation на каждом logging_step\n",
    "    evaluate_during_training : bool = True\n",
    "    # Должно быть True, если Вы используете uncased модели\n",
    "    do_lower_case : bool = True #\n",
    "    \n",
    "    per_gpu_train_batch_size : int = 8 # размер батча для обучения\n",
    "    per_gpu_eval_batch_size : int = 8 # размер батча для eval\n",
    "    learning_rate : float = 5e-5 # learning rate\n",
    "    gradient_accumulation_steps : int = 1 # количество шагов, которые нужно сделать перед backward/update pass\n",
    "    weight_decay : float = 0.0 # weight decay\n",
    "    adam_epsilon : float = 1e-8 # эпсилон для Adam\n",
    "    max_grad_norm : float = 1.0 # максимальная норма градиента\n",
    "    num_train_epochs : float = 5.0 # количество эпох на обучение\n",
    "    max_steps : int = -1 # общее количество шагов на обучение (override num_train_epochs)\n",
    "    warmup_steps : int = 0 # warmup \n",
    "    n_best_size : int = 5 # количество ответов, которые надо сгенерировать для записи в nbest_predictions.json\n",
    "    max_answer_length : int = 30 # максимально возможная длина ответа\n",
    "    verbose_logging : bool = True # печатать или нет warnings, относящиеся к обработке данных\n",
    "    logging_steps : int = 5000 # логировать каждые X шагов\n",
    "    save_steps : int = 5000 # сохранять чекпоинт каждые X шагов\n",
    "        \n",
    "    # Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number\n",
    "    eval_all_checkpoints : bool = True\n",
    "    no_cuda : bool = False # не использовать CUDA\n",
    "    overwrite_output_dir : bool = True # переписывать ли содержимое директории с выходными файлами\n",
    "    overwrite_cache : bool = True # переписывать ли закешированные данные для обучения и evaluation\n",
    "    seed : int = 42 # random seed\n",
    "    local_rank : int = -1 # local rank для распределенного обучения на GPU\n",
    "    fp16 : bool = False # использовать ли 16-bit (mixed) precision (через NVIDIA apex) вместо 32-bit\"\n",
    "    # Apex AMP optimization level: ['O0', 'O1', 'O2', and 'O3'].\n",
    "    # Подробнее тут: https://nvidia.github.io/apex/amp.html\n",
    "    fp16_opt_level : str = '01'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) \\\n",
    "                  for conf in (BertConfig, XLNetConfig, XLMConfig)), ())\n",
    "ALL_MODELS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "args = TRAIN_OPTS()\n",
    "train_dataset = load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train(args, train_dataset, model, tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Сохраняем веса дообученной модели на диск, чтобы в следующий раз не обучать модель заново."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), 'models/bert_squad_final_5epoch.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Подгрузить веса модели можно так:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load('models/bert_squad_5epochs.pt'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Оценка качества работы модели"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH_TO_DEV_SQUAD = 'dev-v2.0.json'\n",
    "PATH_TO_SMALL_DEV_SQUAD = 'small_dev-v2.0.json'\n",
    "\n",
    "with open(PATH_TO_DEV_SQUAD, 'r') as iofile:\n",
    "    full_sample = json.load(iofile)\n",
    "    \n",
    "small_sample = {\n",
    "    'version': full_sample['version'],\n",
    "    'data': full_sample['data'][:1]\n",
    "}\n",
    "\n",
    "with open(PATH_TO_SMALL_DEV_SQUAD, 'w') as iofile:\n",
    "    json.dump(small_sample, iofile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_seq_length = 384\n",
    "outside_pos = max_seq_length + 10\n",
    "doc_stride = 128\n",
    "max_query_length = 64\n",
    "max_answer_length = 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "examples = read_squad_examples(\n",
    "    input_file=PATH_TO_SMALL_DEV_SQUAD,\n",
    "    is_training=False,\n",
    "    version_2_with_negative=True)\n",
    "\n",
    "features = convert_examples_to_features(\n",
    "    examples=examples,\n",
    "    tokenizer=tokenizer,\n",
    "    max_seq_length=max_seq_length,\n",
    "    doc_stride=doc_stride,\n",
    "    max_query_length=max_query_length,\n",
    "    is_training=False,\n",
    "    cls_token_segment_id=0,\n",
    "    pad_token_segment_id=0,\n",
    "    cls_token_at_end=False\n",
    ")\n",
    "\n",
    "input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)\n",
    "input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)\n",
    "segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)\n",
    "cls_index = torch.tensor([f.cls_index for f in features], dtype=torch.long)\n",
    "p_mask = torch.tensor([f.p_mask for f in features], dtype=torch.float)\n",
    "\n",
    "example_index = torch.arange(input_ids.size(0), dtype=torch.long)\n",
    "dataset = TensorDataset(input_ids, input_mask, segment_ids, example_index, cls_index, p_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_sampler = SequentialSampler(dataset)\n",
    "eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_list(tensor):\n",
    "    return tensor.detach().cpu().tolist()\n",
    "\n",
    "all_results = []\n",
    "for idx, batch in enumerate(tqdm.tqdm_notebook(eval_dataloader, desc=\"Evaluating\")):\n",
    "    model.eval()\n",
    "    batch = tuple(t.to(device) for t in batch)\n",
    "    with torch.no_grad():\n",
    "        inputs = {'input_ids':      batch[0],\n",
    "                  'attention_mask': batch[1]\n",
    "                  }\n",
    "        inputs['token_type_ids'] = batch[2]\n",
    "        example_indices = batch[3]\n",
    "        outputs = model(**inputs)\n",
    "\n",
    "    for i, example_index in enumerate(example_indices):\n",
    "        eval_feature = features[example_index.item()]\n",
    "        unique_id = int(eval_feature.unique_id)\n",
    "        result = RawResult(unique_id    = unique_id,\n",
    "                           start_logits = to_list(outputs[0][i]),\n",
    "                           end_logits   = to_list(outputs[1][i]))\n",
    "        all_results.append(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_results[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_best_size = 5\n",
    "do_lower_case = True\n",
    "output_prediction_file = 'output_1best_file'\n",
    "output_nbest_file = 'output_nbest_file'\n",
    "output_na_prob_file = 'output_na_prob_file'\n",
    "verbose_logging = True\n",
    "version_2_with_negative = True\n",
    "null_score_diff_threshold = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Генерируем файл с n лучшими ответами `output_nbest_file`\n",
    "write_predictions(examples, features, all_results, n_best_size,\n",
    "                    max_answer_length, do_lower_case, output_prediction_file,\n",
    "                    output_nbest_file, output_na_prob_file, verbose_logging,\n",
    "                    version_2_with_negative, null_score_diff_threshold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Считаем метрики используя официальный SQuAD script\n",
    "evaluate_options = EVAL_OPTS(data_file=PATH_TO_SMALL_DEV_SQUAD,\n",
    "                             pred_file=output_prediction_file,\n",
    "                             na_prob_file=output_na_prob_file)\n",
    "results = evaluate_on_squad(evaluate_options)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Посмотрим глазами на вопросы и предсказанные БЕРТом ответы:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('output_nbest_file', 'r') as iofile:\n",
    "    predicted_answers = json.load(iofile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions = {}\n",
    "for paragraph in small_sample['data'][0]['paragraphs']:\n",
    "    for question in paragraph['qas']:\n",
    "        questions[question['id']] = {\n",
    "            'question': question['question'],\n",
    "            'answers': question['answers'],\n",
    "            'paragraph': paragraph['context']\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for q_num, (key, data) in enumerate(predicted_answers.items()):\n",
    "    gt = '' if len(questions[key]['answers']) == 0 else questions[key]['answers'][0]['text']\n",
    "    print('Вопрос {0}:'.format(q_num+1), questions[key]['question'])\n",
    "    print('-----------------------------------')\n",
    "    print('Ground truth:', gt)\n",
    "    print('-----------------------------------')   \n",
    "    print('Ответы, предсказанные БЕРТом:')\n",
    "    preds = ['{0}) '.format(ans_num + 1) + answer['text'] + \\\n",
    "             ' (уверенность {0:.2f}%)'.format(answer['probability']*100) \\\n",
    "             for ans_num, answer in enumerate(data)]\n",
    "    print('\\n'.join(preds))\n",
    "#     print('-----------------------------------')   \n",
    "#     print('Параграф:', questions[key]['paragraph'])\n",
    "    print('\\n\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!export SQUAD_DIR=/path/to/SQUAD\n",
    "\n",
    "python run_squad.py \\\n",
    "  --model_type bert \\\n",
    "  --model_name_or_path bert-base-cased \\\n",
    "  --do_train \\\n",
    "  --do_eval \\\n",
    "  --do_lower_case \\\n",
    "  --train_file $SQUAD_DIR/train-v1.1.json \\\n",
    "  --predict_file $SQUAD_DIR/dev-v1.1.json \\\n",
    "  --per_gpu_train_batch_size 12 \\\n",
    "  --learning_rate 3e-5 \\\n",
    "  --num_train_epochs 2.0 \\\n",
    "  --max_seq_length 384 \\\n",
    "  --doc_stride 128 \\\n",
    "  --output_dir /tmp/debug_squad/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "48px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}