NLP/dataprep_bpe.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from tokenizers import decoders, models, normalizers, pre_tokenizers, processors, trainers, Tokenizer\n",
    "from collections import defaultdict\n",
    "import pickle\n",
    "import numpy as np\n",
    "# !wget https://nlp.stanford.edu/projects/nmt/data/wmt14.en-de/train.en -P ../data\n",
    "# !wget https://nlp.stanford.edu/projects/nmt/data/wmt14.en-de/train.de -P ../data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TextLoader:\n",
    "    def __init__(self, PATH):\n",
    "        \n",
    "        self.corpus = open(PATH, 'r').read()\n",
    "        self.sentences = self.corpus.split('\\n')\n",
    "\n",
    "    def remove_rare_words(self,vocab_dict, min_repeat):\n",
    "\n",
    "        wordList = [k for k,v in vocab_dict.items() if v >= min_repeat]\n",
    "        vocab = set(wordList)\n",
    "        print(len(vocab), len(wordList))\n",
    "\n",
    "    def get_all_chars(self):\n",
    "        chars = defaultdict(lambda: 0)\n",
    "        for sen in self.sentences:\n",
    "            for char in sen:\n",
    "                chars[char] += 1\n",
    "\n",
    "        return chars\n",
    "\n",
    "    def remove_sentence_with_rare_chars(self, min_count, dict):\n",
    "        sens_idx = []\n",
    "        for idx, sen in enumerate(self.sentences):\n",
    "            flag = 1\n",
    "            for char in sen:\n",
    "                if(dict[char] < min_count):\n",
    "                    flag = 0\n",
    "                    break\n",
    "            if flag:\n",
    "                sens_idx.append(idx)\n",
    "        return sens_idx\n",
    "\n",
    "    def create_cleaned_data(self, idxs, ext):\n",
    "        cleaned_sentences = [self.sentences[i] for i in idxs]\n",
    "        self.sentences = cleaned_sentences\n",
    "        print(len(self.sentences))\n",
    "\n",
    "        clean_file = open('../data/cleaned_train.' + ext, 'w+')\n",
    "        for i in range(len(self.sentences)):  \n",
    "            clean_file.write(self.sentences[i] + '\\n')\n",
    "            \n",
    "    def batch_iterator(self, batch_size):\n",
    "        for i in range(0, len(self.sentences), batch_size):\n",
    "            yield self.sentences[i : i + batch_size]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "loader_en = TextLoader('../data/train.en')\n",
    "loader_de = TextLoader('../data/train.de')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "chars_en = loader_en.get_all_chars()\n",
    "chars_de = loader_de.get_all_chars()\n",
    "\n",
    "chars = defaultdict(lambda:0)\n",
    "for k,v in chars_en.items():\n",
    "    chars[k] += v\n",
    "for k,v in chars_de.items():\n",
    "    chars[k] += v\n",
    "\n",
    "chars = dict(sorted(chars.items(), key=lambda item: item[1], reverse=True))\n",
    "# dict(sorted(chars_de.items(), key=lambda item: item[1], reverse=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4442493\n",
      "4442493\n"
     ]
    }
   ],
   "source": [
    "cleaned_en = loader_en.remove_sentence_with_rare_chars(1000, chars)\n",
    "cleaned_de = loader_de.remove_sentence_with_rare_chars(1000, chars)\n",
    "cleaned_idx = list(set(cleaned_en) & set(cleaned_de))\n",
    "len(cleaned_idx), len(cleaned_en), len(cleaned_de)\n",
    "\n",
    "loader_de.create_cleaned_data(cleaned_idx, 'de')\n",
    "loader_en.create_cleaned_data(cleaned_idx, 'en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "loader_en = TextLoader('../data/cleaned_train.en')\n",
    "loader_de = TextLoader('../data/cleaned_train.de')\n",
    "\n",
    "assert(len(loader_de.sentences) == len(loader_en.sentences))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Byte Pair Encoding\n",
    " \n",
    "- Starts with a dictionary of individual characters and merges them to create new words upto the max size of vocabulary. \n",
    "- No need for UNK token sice if the word will be tokenized into the largest available words in dictionary and into individual characters in the worst case.\n",
    "- White space information between the words is not preserved."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "tokenizer_en = Tokenizer(models.BPE())\n",
    "tokenizer_en.normalizer = normalizers.Lowercase()\n",
    "# tokenizer_en.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)\n",
    "\n",
    "tokenizer_en.pre_tokenizer = pre_tokenizers.WhitespaceSplit()\n",
    "\n",
    "trainer_en = trainers.BpeTrainer(vocab_size=25000,special_tokens=[\"[PAD]\"])\n",
    "tokenizer_en.train_from_iterator(loader_en.batch_iterator(1000), trainer=trainer_en)\n",
    "\n",
    "# tokenizer_en.decoder =  decoders.ByteLevel()\n",
    "tokenizer_en.save(\"../data/tokenizer_en_25000.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "li = []\n",
    "for batch in loader_en.batch_iterator(1024):\n",
    "    encoding = tokenizer_en.encode_batch(batch)\n",
    "    for i in range(len(encoding)):\n",
    "        li.append(len(encoding[i].ids))\n",
    "\n",
    "li = sorted(li, reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tokenizer_de = Tokenizer(models.BPE())\n",
    "tokenizer_de.normalizer = normalizers.Lowercase()\n",
    "# tokenizer_de.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)\n",
    "\n",
    "tokenizer_de.pre_tokenizer = pre_tokenizers.WhitespaceSplit()\n",
    "\n",
    "# print(tokenizer.pre_tokenizer.pre_tokenize_str(\"Ich liebe das wirklich .\"))\n",
    "\n",
    "trainer_de = trainers.BpeTrainer(vocab_size=25000, special_tokens=[\"[SOS]\",\"[EOS]\",\"[PAD]\"])\n",
    "tokenizer_de.train_from_iterator(loader_de.batch_iterator(1000), trainer=trainer_de)\n",
    "\n",
    "tokenizer_de.post_processor = processors.TemplateProcessing(\n",
    "    single=f\"[SOS]:0 $A:0 [EOS]:0\",\n",
    "    special_tokens=[(\"[SOS]\", 0), (\"[EOS]\", 1)])\n",
    "# tokenizer_de.decoder =  decoders.ByteLevel()\n",
    "\n",
    "tokenizer_de.save(\"../data/tokenizer_de_25000.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_tokens= []\n",
    "encodes_len = []\n",
    "for idx, batch in enumerate(loader_en.batch_iterator(1024)):\n",
    "    encoding = tokenizer_en.encode_batch(batch)\n",
    "    for i in range(len(encoding)):\n",
    "        en_tokens.append(encoding[i].ids)\n",
    "        encodes_len.append(len(encoding[i].ids))\n",
    "\n",
    "de_tokens = []\n",
    "for idx, batch in enumerate(loader_de.batch_iterator(1024)):\n",
    "    encoding = tokenizer_de.encode_batch(batch)\n",
    "    for i in range(len(encoding)):\n",
    "        de_tokens.append(encoding[i].ids)\n",
    "\n",
    "encodes_len , en_tokens, de_tokens = (list(t) for t in zip(*sorted(zip(encodes_len , en_tokens, de_tokens))))\n",
    "\n",
    "\n",
    "# Dump the tokenized lists as pickle files for faster retrieval\n",
    "with open('../data/en_tokenized.pkl', 'wb') as f:\n",
    "    en_tokens = [np.array(i, dtype=np.uint16) for i in en_tokens]\n",
    "    pickle.dump(en_tokens, f)\n",
    "\n",
    "with open('../data/de_tokenized.pkl', 'wb') as f:\n",
    "    de_tokens = [np.array(i, dtype=np.uint16) for i in de_tokens]\n",
    "    pickle.dump(de_tokens, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4442494\n"
     ]
    }
   ],
   "source": [
    "with open('../data/en_tokenized.pkl', 'rb') as f:\n",
    "    en_tokens = pickle.load(f)\n",
    "\n",
    "with open('../data/de_tokenized.pkl', 'rb') as f:\n",
    "    de_tokens = pickle.load(f)\n",
    "\n",
    "print(len(en_tokens))\n",
    "assert(len(de_tokens) == len(en_tokens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer_de = Tokenizer.from_file(\"../data/tokenizer_de_25000.json\")\n",
    "tokenizer_en = Tokenizer.from_file(\"../data/tokenizer_en_25000.json\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "myenv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "vscode": {
   "interpreter": {
    "hash": "9564eaee7b2d18fff668db0d56d5be435fa415a0b75ee906b58019918c1dc466"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}