persian_stt.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    },
    "colab": {
      "name": "persian-stt.ipynb",
      "private_outputs": true,
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f79d99ef"
      },
      "source": [
        "# Train a 🐸 STT model with Common Voice data 💫\n",
        "\n",
        "👋 Hello and welcome to Coqui (🐸) STT \n",
        "\n",
        "This notebook shows a **typical workflow** for **training** and **testing** an 🐸 STT model on data from Common Voice.\n",
        "\n",
        "In this notebook, we will:\n",
        "\n",
        "1. Download Common Voice data (pre-formatted for 🐸 STT)\n",
        "2. Configure the training and testing runs\n",
        "3. Train a new model\n",
        "4. Test the model and display its performance\n",
        "\n",
        "So, let's jump right in!\n",
        "\n",
        "*PS - If you just want a working, off-the-shelf model, check out the [🐸 Model Zoo](https://www.coqui.ai/models)*"
      ],
      "id": "f79d99ef"
    },
    {
      "cell_type": "code",
      "source": [
        "!python -m pip install --upgrade pip wheel setuptools\n",
        "! pip install coqui_stt_training\n",
        "! apt-get install libopusfile0 libopus-dev libopusfile-dev\n",
        "!pip install 'tensorflow-gpu==1.15.4'"
      ],
      "metadata": {
        "id": "PYGWv8Ru1q92"
      },
      "id": "PYGWv8Ru1q92",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "%cd '/content'\n",
        "!git clone https://github.com/coqui-ai/STT.git\n",
        "%cd STT\n",
        "!pip install -r requirements_eval_tflite.txt\n",
        "!pip install -r requirements_tests.txt\n",
        "!pip install -r requirements_transcribe.txt\n",
        "!python3 setup.py bdist_wheel\n",
        "!pip install dist/*.whl"
      ],
      "metadata": {
        "id": "w8fqu5x2Rff-"
      },
      "id": "w8fqu5x2Rff-",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "import os\n",
        "os.chdir('/content')\n",
        "drive.mount('drive', force_remount=True)\n",
        "gdrive = '/content/drive/MyDrive/'"
      ],
      "metadata": {
        "id": "0iCk91sw8qB9"
      },
      "id": "0iCk91sw8qB9",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# for mounting 2 diff drives, from https://stackoverflow.com/questions/53728127/mount-multiple-drives-in-google-colab\n",
        "!sudo add-apt-repository -y ppa:alessandro-strada/ppa 2>&1 > /dev/null\n",
        "!sudo apt-get update -qq 2>&1 > /dev/null\n",
        "!sudo apt -y install -qq google-drive-ocamlfuse 2>&1 > /dev/null\n",
        "!google-drive-ocamlfuse"
      ],
      "metadata": {
        "id": "3E9p5-VrHJKM"
      },
      "id": "3E9p5-VrHJKM",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!apt-get install -qq w3m\n",
        "!xdg-settings set default-web-browser w3m.desktop # to set default browser\n",
        "%cd /content\n",
        "!mkdir drive2\n",
        "%cd drive2\n",
        "!mkdir MyDrive\n",
        "%cd ..\n",
        "!google-drive-ocamlfuse /content/drive2/MyDrive\n",
        "gdrive2 = '/content/drive2/MyDrive/'\n",
        "!cp 'drive2/MyDrive/Voice-Cloning/cpg' '/usr/local/bin'\n",
        "!chmod +x '/usr/local/bin/cpg'"
      ],
      "metadata": {
        "id": "oNEvcHiiHRW-"
      },
      "id": "oNEvcHiiHRW-",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "gdrive = '/content/drive/MyDrive/'\n",
        "gdrive2 = '/content/drive2/MyDrive/'"
      ],
      "metadata": {
        "id": "3HEqVibKayAn"
      },
      "id": "3HEqVibKayAn",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!ln -s '/content/drive2/MyDrive/' /content/drive2/My\\ Drive"
      ],
      "metadata": {
        "id": "KogQ43HVKPea"
      },
      "id": "KogQ43HVKPea",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        ""
      ],
      "metadata": {
        "id": "U-wMjs1T_OTR"
      },
      "id": "U-wMjs1T_OTR"
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content\n",
        "!cpg '$gdrive2''Voice-Cloning/clips.tar.gz' '/content'\n",
        "!tar -xf clips.tar.gz\n",
        "!rm clips.tar.gz\n",
        "!rm clips/dev.csv clips/train.csv clips/train-all.csv clips/other.csv clips/test.csv"
      ],
      "metadata": {
        "id": "bmlK4RyFF36C"
      },
      "id": "bmlK4RyFF36C",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# generate the training data\n",
        "# !apt-get install sox libsox-fmt-mp3\n",
        "# !python3 STT/bin/import_cv2.py --filter_alphabet /content/alphabet.txt /content/cv-corpus-9.0-2022-04-27/fa/"
      ],
      "metadata": {
        "id": "oHKFgnWL5xl5"
      },
      "id": "oHKFgnWL5xl5",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%tensorflow_version 1.x\n",
        "import tensorflow as tf\n",
        "device_name = tf.test.gpu_device_name()\n",
        "if device_name != '/device:GPU:0':\n",
        "  raise SystemError('GPU device not found')\n",
        "print('Found GPU at: {}'.format(device_name))"
      ],
      "metadata": {
        "id": "GcPrjjNz3leP"
      },
      "id": "GcPrjjNz3leP",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!wc -l clips/*.csv"
      ],
      "metadata": {
        "id": "gGSTMYVadIca"
      },
      "id": "gGSTMYVadIca",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!nvidia-smi -L"
      ],
      "metadata": {
        "id": "xTf-KHjw2Zw3"
      },
      "id": "xTf-KHjw2Zw3",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataset_dir = '/content/clips/'\n",
        "metadata = dataset_dir+'validated.csv'\n",
        "train_files = dataset_dir+'train.csv'\n",
        "dev_files = dataset_dir+\"dev.csv\"\n",
        "test_files = dataset_dir+\"test.csv\"\n",
        "checkpoint_dir = gdrive+'Voice-Cloning/STT/checkpoints'\n",
        "alphabet_file = gdrive+'Voice-Cloning/'+'alphabet.txt'\n",
        "export_dir = gdrive+'Voice-Cloning/STT/models'"
      ],
      "metadata": {
        "id": "mGnD6IWZXDjR"
      },
      "id": "mGnD6IWZXDjR",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fa2aec77"
      },
      "source": [
        "!ls clips | wc -l"
      ],
      "id": "fa2aec77",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# !wget https://github.com/mozilla/DeepSpeech/releases/download/v0.9.3/deepspeech-0.9.3-checkpoint.tar.gz\n",
        "# !tar -xf deepspeech-0.9.3-checkpoint.tar.gz\n",
        "# for transfer learning"
      ],
      "metadata": {
        "id": "9eoJIh7JZObO"
      },
      "id": "9eoJIh7JZObO",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "03b33d2b"
      },
      "source": [
        "!cpg '$gdrive2''Voice-Cloning/kenlm-persian.scorer' '/content/kenlm-persian.scorer'"
      ],
      "id": "03b33d2b",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d9dfac21"
      },
      "source": [
        "## ✅ Configure & set hyperparameters\n",
        "\n",
        "Coqui STT comes with a long list of hyperparameters you can tweak. We've set default values, but you will often want to set your own. You can use `initialize_globals_from_args()` to do this. \n",
        "\n",
        "You must **always** configure the paths to your data, and you must **always** configure your alphabet. Additionally, here we show how you can specify the size of hidden layers (`n_hidden`), the number of epochs to train for (`epochs`), and to initialize a new model from scratch (`load_train=\"init\"`).\n",
        "\n",
        "If you're training on a GPU, you can uncomment the (larger) training batch sizes for faster training."
      ],
      "id": "d9dfac21"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d264fdec"
      },
      "source": [
        "from coqui_stt_training.util.config import initialize_globals_from_args\n",
        "# !rm STT/alphabet.txt\n",
        "initialize_globals_from_args(\n",
        "    scorer_path='/content/kenlm-persian.scorer',\n",
        "    # train_files=[train_files],\n",
        "    # dev_files=[dev_files],\n",
        "    # test_files=[test_files],\n",
        "    dropout_rate=0.175,\n",
        "    # load_checkpoint_dir='/content/deepspeech-0.9.3-checkpoint',\n",
        "    # drop_source_layers=2, # set when tranfer learning\n",
        "    learning_rate=0.000095,\n",
        "    force_initialize_learning_rate=True,\n",
        "    train_cudnn=True,\n",
        "    reduce_lr_on_plateau=True,\n",
        "    plateau_epochs=3,\n",
        "    plateau_reduction=0.2,\n",
        "    auto_input_dataset=metadata,\n",
        "    # alphabet_config_path=alphabet_file,\n",
        "    checkpoint_dir=checkpoint_dir,\n",
        "    export_dir=export_dir,\n",
        "    epochs=200,\n",
        "    train_batch_size=128,\n",
        "    dev_batch_size=128,\n",
        "    test_batch_size=384,\n",
        ")"
      ],
      "id": "d264fdec",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ae82fd75"
      },
      "source": [
        "## ✅ Train a new model\n",
        "\n",
        "Let's kick off a training run 🚀🚀🚀 (using the configure you set above)."
      ],
      "id": "ae82fd75"
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "550a504e"
      },
      "source": [
        "from coqui_stt_training.train import train\n",
        "\n",
        "train()"
      ],
      "id": "550a504e",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# view loss graph\n",
        "%load_ext tensorboard\n",
        "%tensorboard --logdir '$checkpoint_dir/summaries/'"
      ],
      "metadata": {
        "id": "UWVFuoGZOjKx"
      },
      "id": "UWVFuoGZOjKx",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9f6dc959"
      },
      "source": [
        "## ✅ Test the model\n",
        "\n",
        "We made it! 🙌\n",
        "\n",
        "Let's kick off the testing run, which displays performance metrics.\n",
        "\n",
        "The settings we used here are for demonstration purposes, so you don't want to deploy this model into production. In this notebook we're focusing on the workflow itself, so it's forgivable 😇\n",
        "\n",
        "You can still train a more State-of-the-Art model by finding better hyperparameters, so go for it 💪"
      ],
      "id": "9f6dc959"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dd42bc7a"
      },
      "source": [
        "from coqui_stt_training.evaluate import test\n",
        "\n",
        "test()"
      ],
      "id": "dd42bc7a",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_name = 'persian_stt'\n",
        "version = '0.1.0'"
      ],
      "metadata": {
        "id": "AC7CznWCrPNs"
      },
      "id": "AC7CznWCrPNs",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!curl -L https://github.com/coqui-ai/STT/releases/download/v0.9.3/convert_graphdef_memmapped_format.linux.amd64.zip | funzip > convert_graphdef_memmapped_format\n",
        "!chmod +x convert_graphdef_memmapped_format "
      ],
      "metadata": {
        "id": "qzw_7B9Wkf75"
      },
      "id": "qzw_7B9Wkf75",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!python -m coqui_stt_training.export \\\n",
        "  --checkpoint_dir='$checkpoint_dir' \\\n",
        "  --export_dir='$export_dir' \\\n",
        "  --export_model_name='$model_name' \\\n",
        "  --export_author_id='oct4pie' \\\n",
        "  --export_model_version='$version' \\\n",
        "  --export_contact_info='https://github.com/Oct4Pie/persian-stt/issues' \\\n",
        "  --export_license='LGPL-3.0-only' \\\n",
        "  --export_language='fa-IR' \\\n",
        "  --export_file_name='$model_name'\n",
        "  \n",
        "# export protocol buffer\n",
        "!python -m coqui_stt_training.export \\\n",
        "  --checkpoint_dir='$checkpoint_dir' \\\n",
        "  --export_dir='$export_dir' \\\n",
        "  --export_tflite='false' \\\n",
        "  --export_model_name='$model_name' \\\n",
        "  --export_author_id='oct4pie' \\\n",
        "  --export_model_version='$version' \\\n",
        "  --export_contact_info='https://github.com/Oct4Pie/persian-stt/issues' \\\n",
        "  --export_license='LGPL-3.0-only' \\\n",
        "  --export_language='fa-IR' \\\n",
        "  --export_file_name='$model_name'"
      ],
      "metadata": {
        "id": "EEPR8lPMocL_"
      },
      "id": "EEPR8lPMocL_",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!./convert_graphdef_memmapped_format --in_graph=\"$export_dir/\"'$model_name''.pb' --out_graph='$export_dir/''$model_name''.pbmm'"
      ],
      "metadata": {
        "id": "89rMLh3CmOdz"
      },
      "id": "89rMLh3CmOdz",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install stt ffmpeg-python"
      ],
      "metadata": {
        "id": "EgAtQAv2JhHZ"
      },
      "id": "EgAtQAv2JhHZ",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# transcription test\n",
        "from stt import Model\n",
        "from io import BytesIO\n",
        "import ffmpeg\n",
        "import numpy as np\n",
        "import wave\n",
        "\n",
        "audio = open('/content/test.mp3','rb').read()\n",
        "\n",
        "out, err = (\n",
        "        ffmpeg.input(\"pipe:0\")\n",
        "        .output(\n",
        "            \"pipe:1\",\n",
        "            f=\"WAV\",\n",
        "            acodec=\"pcm_s16le\",\n",
        "            ac=1,\n",
        "            ar=\"16k\",\n",
        "            loglevel=\"error\",\n",
        "            hide_banner=None,\n",
        "        )\n",
        "        .run(input=audio, capture_stdout=True, capture_stderr=True)\n",
        "    )\n",
        "if err:\n",
        "  raise Exception(err)\n",
        "\n",
        "with wave.Wave_read(BytesIO(out)) as wav:\n",
        "  audio = np.frombuffer(wav.readframes(wav.getnframes()), np.int16)\n",
        "ds = Model(export_dir+'/'+model_name+'.tflite')\n",
        "txt = ds.stt(audio)\n",
        "\n",
        "with open('transcript.txt', 'w') as f:\n",
        "  f.write(txt)"
      ],
      "metadata": {
        "id": "xhk9MA--nbEJ"
      },
      "id": "xhk9MA--nbEJ",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://storage.googleapis.com/danielk-files/farsi-text/merged_files/commoncrawl_fa_merged.txt # dirty text corpus"
      ],
      "metadata": {
        "id": "39Cq-D6ywFcv"
      },
      "id": "39Cq-D6ywFcv",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!wc -l commoncrawl_fa_merged.txt"
      ],
      "metadata": {
        "id": "BvDe_0mJy8Mx"
      },
      "id": "BvDe_0mJy8Mx",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install hazm"
      ],
      "metadata": {
        "id": "6_w24fu0BIrI"
      },
      "id": "6_w24fu0BIrI",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# clean up the text\n",
        "import re\n",
        "import unicodedata\n",
        "import sys\n",
        "from urllib.parse import unquote\n",
        "from html.parser import HTMLParser\n",
        "from hazm import WordTokenizer, word_tokenize\n",
        "\n",
        "tokenizer = WordTokenizer(join_verb_parts=False)\n",
        "\n",
        "def persianify(sentence):\n",
        "  # omit short phrases/sentences\n",
        "  if len(sentence.split()) < 5:\n",
        "    return\n",
        "  \n",
        "  # b'\\xd9\\x8e\\xd9\\x90\\xd9\\x8f\\xd9\\x8d\\xd9\\x8b'.decode()\n",
        "  alpha = ' !\"&\\'()-.:=،؛؟ءآأؤئابةتثجحخدذرزسشصضطظعغـفقكلمنهوىئًٌَُِّْ٬پچژکگۀیے–“”…'\n",
        "  puncs = '،؛؟!:.()\"\\'…“”'\n",
        "\n",
        "  try:\n",
        "    for letter in sentence:\n",
        "      if letter not in alpha:\n",
        "        return\n",
        "\n",
        "    sentence = sentence.strip().replace('...', '')\n",
        "    l_seg = sorted(sentence.split(), key=lambda w: len(w))[-1]\n",
        "    if len(l_seg) > 11:\n",
        "      l_seg = sorted(word_tokenize(l_seg), key=lambda w: len(w))[-1]\n",
        "      if len(l_seg) > 12:\n",
        "        return\n",
        "\n",
        "    sentence = ' '.join(sentence.split('-'))\n",
        "    segs = tokenizer.tokenize(sentence)\n",
        "\n",
        "    if segs[-1] == '.':\n",
        "      del segs[-1]\n",
        "      segs.insert(0, '.')\n",
        "\n",
        "    sentence = \"\"\n",
        "    for seg in segs:\n",
        "      # sentence += seg if seg in puncs else ' '+seg\n",
        "      if seg in puncs:\n",
        "        sentence += seg\n",
        "\n",
        "      else:\n",
        "        sentence += ' '+seg\n",
        "\n",
        "      \n",
        "\n",
        "    \n",
        "    if sentence[-1] in ':،\"\\'…“”':\n",
        "      sentence.replace(sentence[-1], '', len(sentence)-4)\n",
        "    return sentence.replace('. ', '.', len(sentence)-4).strip().replace('( ',' (')\n",
        "\n",
        "  except Exception as e:\n",
        "    print('error: ', e, f'for \"{sentence.split()}\"')\n",
        "    return\n",
        "\n",
        "# from https://github.com/common-voice/CorporaCreator\n",
        "\n",
        "RE_DIGITS = re.compile('\\d')\n",
        "\n",
        "def _has_digit(sentence):\n",
        "    return RE_DIGITS.search(sentence)\n",
        "\n",
        "\n",
        "class _HTMLStripper(HTMLParser):\n",
        "    \"\"\"Class that strips HTML from strings.\n",
        "    Examples:\n",
        "        >>> stripper = _HTMLStripper()\n",
        "        >>> stripper.feed(html)\n",
        "        >>> nohtml = stripper.get_data()\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.reset()\n",
        "        self.strict = False\n",
        "        self.convert_charrefs = True\n",
        "        self.fed = []\n",
        "\n",
        "    def handle_data(self, d):\n",
        "        self.fed.append(d)\n",
        "\n",
        "    def get_data(self):\n",
        "        return \"\".join(self.fed)\n",
        "\n",
        "\n",
        "def _strip_tags(html):\n",
        "    \"\"\"Removes HTML tags from passed text.\n",
        "    Args:\n",
        "      html (str): String containing HTML\n",
        "    Returns:\n",
        "      (str): String with HTML removed\n",
        "    \"\"\"\n",
        "    s = _HTMLStripper()\n",
        "    s.feed(html)\n",
        "    return s.get_data()\n",
        "\n",
        "\n",
        "def _strip_string(sentence):\n",
        "    \"\"\"Cleans a string based on a whitelist of printable unicode categories.\n",
        "    You can find a full list of categories here:\n",
        "    http://www.fileformat.info/info/unicode/category/index.htm\n",
        "    \"\"\"\n",
        "    letters     = ('LC', 'Ll', 'Lm', 'Lo', 'Lt', 'Lu')\n",
        "    numbers     = ('Nd', 'Nl', 'No')\n",
        "    marks       = ('Mc', 'Me', 'Mn')\n",
        "    punctuation = ('Pc', 'Pd', 'Pe', 'Pf', 'Pi', 'Po', 'Ps')\n",
        "    symbol      = ('Sc', 'Sk', 'Sm', 'So')\n",
        "    space       = ('Zs',)\n",
        "\n",
        "    allowed_categories = letters + numbers + marks + punctuation + symbol + space\n",
        "\n",
        "    return u''.join([c for c in sentence if unicodedata.category(c) in allowed_categories])\n",
        "\n",
        "\n",
        "def common(sentence):\n",
        "    \"\"\"Cleans up the passed sentence in a language independent manner, removing or reformatting invalid data.\n",
        "    Args:\n",
        "      sentence (str): Sentence to be cleaned up.\n",
        "    Returns:\n",
        "      (is_valid,str): A boolean indicating validity and cleaned up sentence.\n",
        "    \"\"\"\n",
        "\n",
        "    # Define a boolean indicating validity\n",
        "    is_valid = True\n",
        "    # Decode any URL encoded elements of sentence\n",
        "    sentence = unquote(sentence)\n",
        "    # Remove any HTML tags\n",
        "    sentence = _strip_tags(sentence)\n",
        "    # Remove non-printable characters\n",
        "    sentence = _strip_string(sentence)\n",
        "    # collapse all whitespace and replace with single space\n",
        "    sentence = (' ').join(sentence.split())\n",
        "    # TODO: Clean up data in a language independent manner\n",
        "    # If the sentence contains digits reject it\n",
        "    if _has_digit(sentence):\n",
        "        is_valid = False\n",
        "    # If the sentence is blank reject it\n",
        "    if not sentence.strip():\n",
        "        is_valid = False\n",
        "    return (is_valid, sentence)"
      ],
      "metadata": {
        "id": "DQnH-ncv93oM"
      },
      "id": "DQnH-ncv93oM",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import time\n",
        "os.chdir('/content')\n",
        "import sys\n",
        "clean = open('clean.txt', 'w')\n",
        "with open('commoncrawl_fa_merged.txt', 'r') as f:\n",
        "  try:\n",
        "    line = next(f)\n",
        "    while line:\n",
        "      for sentence in line.split('\\t'):\n",
        "        s = common(sentence.replace('\\n', '').replace('/','').replace('|',''))[1]\n",
        "        s = persianify(s)\n",
        "        if s:\n",
        "          clean.write(s+'\\n')\n",
        "\n",
        "      line = next(f)\n",
        "  except Exception as e:\n",
        "      clean.close()\n",
        "      print('error: ', e, f'for {line}')\n",
        "\n",
        "\n",
        "clean.close()"
      ],
      "metadata": {
        "id": "L32aLw1rBw8u"
      },
      "id": "L32aLw1rBw8u",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!head -n 6 clean.txt"
      ],
      "metadata": {
        "id": "JrXEgcy8Zut6"
      },
      "id": "JrXEgcy8Zut6",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "if not os.getcwd().endswith('STT'):\n",
        "  os.chdir('STT')\n",
        "# !pip uninstall coqui_stt_training\n",
        "!echo  \"$checkpoint_dir\"\n",
        "!python3 lm_optimizer.py \\\n",
        "  --checkpoint_dir \"$checkpoint_dir\" \\\n",
        "  --test_files \"$test_files\""
      ],
      "metadata": {
        "id": "rJuXDYViz5wN"
      },
      "id": "rJuXDYViz5wN",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!cpg '/content/clean.txt' '$gdrive''Voice-Cloning/'\n",
        "!cpg '$gdrive2''Voice-Cloning/clean-nodup-nopunc.txt' .\n",
        "!cpg '$gdrive''Voice-Cloning/clean-nodup.txt' ."
      ],
      "metadata": {
        "id": "exXuKX8eirPm"
      },
      "id": "exXuKX8eirPm",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!sort --unique -o clean-nodup.txt clean.txt # remove duplicates (python failed because of high memory usage)"
      ],
      "metadata": {
        "id": "kToyZgRzrxRG"
      },
      "id": "kToyZgRzrxRG",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# remove special chars\n",
        "def rm_spec(sentence):\n",
        "  # puncs = '،؛؟!:.()\"\\'…“”–&-='\n",
        "  puncs = '()\"\\'…“”–&-='\n",
        "  for punc in puncs:\n",
        "    sentence = sentence.replace(punc, '')\n",
        "  return sentence\n",
        "\n",
        "ofile = open('clean-nodup-nopspec.txt', 'w')\n",
        "ifile = open('clean-nodup.txt')\n",
        "\n",
        "for line in ifile:\n",
        "  phrase = rm_spec(line).strip()\n",
        "  if len(phrase) > 4:\n",
        "    ofile.write(phrase+'\\n')\n",
        "\n",
        "ofile.close()"
      ],
      "metadata": {
        "id": "Xtm7RCo3KWR3"
      },
      "id": "Xtm7RCo3KWR3",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# !sort --unique -r -o clean-nodup-nopspec1.txt clean-nodup-nopspec.txt\n",
        "!ls"
      ],
      "metadata": {
        "id": "264-7sioMChG"
      },
      "id": "264-7sioMChG",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!cpg clean-nodup-nospec.txt '$gdrive2''Voice-Cloning/clean-nodup-nospec.txt'"
      ],
      "metadata": {
        "id": "WLukZr1w462k"
      },
      "id": "WLukZr1w462k",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# rm all punctuations for scorer\n",
        "def rm_punc(sentence):\n",
        "  puncs = '،؛؟!:.()\"\\'…“”–&-=ـِء'\n",
        "  # puncs = '()\"\\'…“”–&-='\n",
        "  for punc in puncs:\n",
        "    sentence = sentence.replace(punc, '')\n",
        "  return sentence\n",
        "\n",
        "ofile = open('clean-nodup-nopunc.txt', 'w')\n",
        "ifile = open('clean-nodup.txt')\n",
        "\n",
        "for line in ifile:\n",
        "  phrase = rm_punc(line).strip()\n",
        "  if len(phrase.split()) > 4:\n",
        "    ofile.write(phrase+'\\n')\n",
        "\n",
        "ofile.close()"
      ],
      "metadata": {
        "id": "ohvBy1FS5BWv"
      },
      "id": "ohvBy1FS5BWv",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!cpg clean-nodup-nopunc.txt '$gdrive2''Voice-Cloning/clean-nodup-nopunc.txt'"
      ],
      "metadata": {
        "id": "ZJHA4_1g5pzd"
      },
      "id": "ZJHA4_1g5pzd",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "## building kenlm\n",
        "# %cd /content/STT/native_client/kenlm\n",
        "# !rm -rf * && \\\n",
        "# \tgit clone https://github.com/kpu/kenlm \n",
        "# %cd kenlm\n",
        "# !git checkout 87e85e66c99ceff1fab2500a7c60c01da7315eec\n",
        "# !mkdir -p build\n",
        "# %cd build\n",
        "# !cmake ..\n",
        "# !make -j $(nproc)"
      ],
      "metadata": {
        "id": "7Ia-LPyszhgE"
      },
      "id": "7Ia-LPyszhgE",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!mv /content/STT/data/lm/persian-scorer '$gdrive''Voice-Cloning/persian-scorer'\n",
        "!cpg -a '$gdrive2''Voice-Cloning/persian-scorer/.' /content/STT/data/lm/persian-scorer"
      ],
      "metadata": {
        "id": "0Asq1pdvxw_d"
      },
      "id": "0Asq1pdvxw_d",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content/STT/data/lm\n",
        "!mkdir 'persian-scorer'\n",
        "!python3 generate_lm.py \\\n",
        "  --input_txt /content/clean-nodup-nopunc.txt \\\n",
        "  --output_dir persian-scorer \\\n",
        "  --kenlm_bins /content/STT/native_client/kenlm/kenlm/build/bin \\\n",
        "  --arpa_order 5 --max_arpa_memory \"85%\" --arpa_prune \"0|0|1\" \\\n",
        "  --binary_a_bits 255 --binary_q_bits 8 --binary_type trie \\\n",
        "  --top_k 500000"
      ],
      "metadata": {
        "id": "V72JqJipjN7H"
      },
      "id": "V72JqJipjN7H",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content/STT/data/lm\n",
        "!curl -L https://github.com/coqui-ai/STT/releases/download/v1.3.0/native_client.tflite.Linux.tar.xz | tar -Jxvf -"
      ],
      "metadata": {
        "id": "Gx8xhRTWFD1b"
      },
      "id": "Gx8xhRTWFD1b",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content/STT/data/lm\n",
        "!./generate_scorer_package \\\n",
        "  --checkpoint \"$checkpoint_dir\" \\\n",
        "  --lm \"/content/STT/data/lm/persian-scorer/lm.binary\" \\\n",
        "  --vocab \"/content/STT/data/lm/persian-scorer/vocab-500000.txt\" \\\n",
        "  --package kenlm-persian.scorer \\\n",
        "  --default_alpha 0.36669178512950323 \\\n",
        "  --default_beta 0.3457913671678824"
      ],
      "metadata": {
        "id": "4OYJvtclPAU_"
      },
      "id": "4OYJvtclPAU_",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content/STT\n",
        "!python3 lm_optimizer.py \\\n",
        "     --test_files \"$test_files\" \\\n",
        "     --checkpoint_dir \"$checkpoint_dir\" \\\n",
        "     --scorer_path \"data/lm/kenlm-persian.scorer\" \\\n",
        "     --n_hidden 2048 \\\n",
        "     --lm_alpha_max 1 \\\n",
        "     --lm_beta_max 1 \\\n",
        "     --n_trials 100 \\\n",
        "     --test_batch_size 384 \\\n",
        "     --train_cudnn 'true'"
      ],
      "metadata": {
        "id": "TTWTsKvLk91y"
      },
      "id": "TTWTsKvLk91y",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!cpg kenlm-persian.scorer '$gdrive2''Voice-Cloning/kenlm-persian.scorer'"
      ],
      "metadata": {
        "id": "-cyXqvM8zCfo"
      },
      "id": "-cyXqvM8zCfo",
      "execution_count": null,
      "outputs": []
    }
  ]
}