Automatic_Text_Language_Identification_In_Python.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Automatic Text Language Identification In Python",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# **Automatic Text Language Identification In Python**.  \n",
        "> https://zoumanakeita.medium.com/  "
      ],
      "metadata": {
        "id": "n_Hi3ryJdObG"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## LangDetect"
      ],
      "metadata": {
        "id": "9K3kU91_dwSi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Install the library\n",
        "!pip install langdetect\n",
        "from langdetect import detect, detect_langs"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y5yebQjcJ_sF",
        "outputId": "9946167a-e12c-491c-fd41-c5882ff23ad3"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting langdetect\n",
            "  Downloading langdetect-1.0.9.tar.gz (981 kB)\n",
            "\u001b[?25l\r\u001b[K     |▍                               | 10 kB 15.0 MB/s eta 0:00:01\r\u001b[K     |▊                               | 20 kB 17.8 MB/s eta 0:00:01\r\u001b[K     |█                               | 30 kB 21.4 MB/s eta 0:00:01\r\u001b[K     |█▍                              | 40 kB 12.7 MB/s eta 0:00:01\r\u001b[K     |█▊                              | 51 kB 5.6 MB/s eta 0:00:01\r\u001b[K     |██                              | 61 kB 6.5 MB/s eta 0:00:01\r\u001b[K     |██▍                             | 71 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |██▊                             | 81 kB 6.1 MB/s eta 0:00:01\r\u001b[K     |███                             | 92 kB 6.8 MB/s eta 0:00:01\r\u001b[K     |███▍                            | 102 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███▊                            | 112 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████                            | 122 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████▍                           | 133 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████▊                           | 143 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████                           | 153 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████▍                          | 163 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████▊                          | 174 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████                          | 184 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████▍                         | 194 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████▊                         | 204 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████                         | 215 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████▍                        | 225 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████▊                        | 235 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████                        | 245 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████▍                       | 256 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████▊                       | 266 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████████                       | 276 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████████▍                      | 286 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████████▊                      | 296 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████████                      | 307 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 317 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 327 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████████                     | 337 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████████▍                    | 348 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████████▊                    | 358 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████████                    | 368 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████████▍                   | 378 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████████▊                   | 389 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 399 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████████████▍                  | 409 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████████████▊                  | 419 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 430 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████████████▍                 | 440 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████████████▊                 | 450 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 460 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 471 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 481 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████████████                | 491 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████████████▍               | 501 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 512 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 522 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 532 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████▊              | 542 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 552 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████▍             | 563 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████▊             | 573 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 583 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████▍            | 593 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 604 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 614 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████▍           | 624 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████▊           | 634 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 645 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▍          | 655 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▊          | 665 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 675 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▍         | 686 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 696 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 706 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 716 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▊        | 727 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 737 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▍       | 747 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 757 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 768 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▍      | 778 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▊      | 788 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 798 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 808 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 819 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 829 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▍    | 839 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 849 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 860 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 870 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 880 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 890 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▍  | 901 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▊  | 911 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 921 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▍ | 931 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▊ | 942 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 952 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▍| 962 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▊| 972 kB 7.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 981 kB 7.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from langdetect) (1.15.0)\n",
            "Building wheels for collected packages: langdetect\n",
            "  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for langdetect: filename=langdetect-1.0.9-py3-none-any.whl size=993242 sha256=bf6eb9e9f23a092ba1a6f390856ac1b4985298ae93a68b0457f8e9b6a18060ea\n",
            "  Stored in directory: /root/.cache/pip/wheels/c5/96/8a/f90c59ed25d75e50a8c10a1b1c2d4c402e4dacfa87f3aff36a\n",
            "Successfully built langdetect\n",
            "Installing collected packages: langdetect\n",
            "Successfully installed langdetect-1.0.9\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def language_detection(text, method = \"single\"):\n",
        "\n",
        "  \"\"\"\n",
        "  @desc: \n",
        "    - detects the language of a text\n",
        "  @params:\n",
        "    - text: the text which language needs to be detected\n",
        "    - method: detection method: \n",
        "      single: if the detection is based on the first option (detect)\n",
        "  @return:\n",
        "    - the langue/list of languages\n",
        "  \"\"\"\n",
        "\n",
        "  if(method.lower() != \"single\"):\n",
        "    result = detect_langs(text)\n",
        "\n",
        "  else:\n",
        "    result = detect(text)\n",
        "\n",
        "  return result"
      ],
      "metadata": {
        "id": "G0mcKmX0J_RH"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Executing the following two cells, you might get different results each time, because of the non-deterministic aspect.\n"
      ],
      "metadata": {
        "id": "gFXmYwfAq7BF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Test first case \n",
        "text = \"\"\"\n",
        "This library is the direct port of Google's language-detection library from Java to Python. \n",
        "Elle est vraiment éfficace dans la détection de langue.\n",
        "\"\"\"\n",
        "print(language_detection(text))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "777iDeA_J_ON",
        "outputId": "8e61850a-0afc-45e7-be5c-aafa8c0f5838"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "en\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Test second case\n",
        "print(language_detection(text, \"all languages\"))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_orcvivbrq2y",
        "outputId": "4a1a1a70-3b18-43f9-ba91-86a3d38133de"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[en:0.714282782965898, fr:0.28571541584211635]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langdetect import DetectorFactory\n",
        "DetectorFactory.seed = 0\n",
        "\n",
        "# Test first case \n",
        "text = \"\"\"\n",
        "This library is the direct port of Google's language-detection library from Java to Python. Developed by Nakatani Shuyo at Cybozu Labs, Inc in 2010, this library has about 99% precision for over 50 languages.\n",
        "\"\"\"\n",
        "print(language_detection(text))\n",
        "\n",
        "# Test second case\n",
        "print(language_detection(text, \"show_proba\"))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DgOWHq23J_F7",
        "outputId": "831c71d0-176e-4f9f-b8dd-eba139cf5caa"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "en\n",
            "[en:0.9999977497863055]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Spacy-langdetect"
      ],
      "metadata": {
        "id": "ZV-XQ4iiJ1bh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#!pip install spacy-langdetect\n",
        "import spacy\n",
        "from spacy_langdetect import LanguageDetector "
      ],
      "metadata": {
        "id": "UQs4tJQwKBCc"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# https://www.actuia.com/english/africa-launch-of-the-initiative-for-the-development-of-artificial-intelligence-in-french-speaking-african-countries/\n",
        "\n",
        "def spacy_language_detection(text, model):\n",
        "\n",
        "  pipeline = list(dict(model.pipeline).keys())\n",
        "\n",
        "  if(not \"language_detector\" in pipeline):\n",
        "    model.add_pipe(LanguageDetector(), name = \"language_detector\", last=True)\n",
        "    \n",
        "  doc = model(text)\n",
        "\n",
        "  return doc._.language"
      ],
      "metadata": {
        "id": "cdC1HbXAKA7r"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "english_text = \"\"\"Niyel, a Dakar-based company that designs, implements, \n",
        "and evaluates advocacy campaigns to change policies, behaviors, and practices, \n",
        "will support the researchers in using the results to influence the implementation of AI-friendly policies.\n",
        "\"\"\"\n",
        "french_text = \"\"\"Intelligence artificielle : la solution pour améliorer l'accès au crédit en Afrique ? \n",
        "Déjà une réalité au Kenya, en Afrique du Sud et au Nigeria, l'évaluation du risque crédit via \n",
        "l'intelligence artificielle dispose d'un fort potentiel en Afrique de l'Ouest malgré les inquiétudes liées à la protection de la vie privée.\"\"\"\n"
      ],
      "metadata": {
        "id": "33DQ-6n6KA4p"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pre_trained_model = spacy.load(\"en_core_web_sm\")"
      ],
      "metadata": {
        "id": "IiVBCR1sDhob"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Detection on English text\n",
        "print(spacy_language_detection(english_text, pre_trained_model))\n",
        "\n",
        "# Detection on French text\n",
        "print(spacy_language_detection(french_text, pre_trained_model))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KBibB2qf6MJm",
        "outputId": "ed546f95-b550-485e-a4cf-73e96e0c922f"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'language': 'en', 'score': 0.9999968055183488}\n",
            "{'language': 'fr', 'score': 0.9999948438951478}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1V7NpNQUDT1j",
        "outputId": "fba02def-d3f2-49bc-b97d-c085e56ebdbf"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['tagger', 'parser', 'ner', 'language_detector']"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## fastText"
      ],
      "metadata": {
        "id": "Mu1L2VmtJ5pY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install fasttext\n",
        "import fasttext as ft"
      ],
      "metadata": {
        "id": "q1gnHd0XKB8L"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the pretrained model\n",
        "ft_model = ft.load_model(\"./pretrained_model/lid.176.bin\")\n",
        "\n",
        "def fasttext_language_predict(text, model = ft_model):\n",
        "\n",
        "  text = text.replace('\\n', \" \")\n",
        "  prediction = model.predict([text])\n",
        "\n",
        "  return prediction"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WV53S6EGKB0P",
        "outputId": "84b9ecb1-0580-4e3a-d62c-4102775f89d5"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(fasttext_language_predict(english_text))\n",
        "print(fasttext_language_predict(french_text))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HwM2Cd5VKBxn",
        "outputId": "14f9dc39-5e4f-4e68-b578-63839e29bc07"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "([['__label__en']], [array([0.8957091], dtype=float32)])\n",
            "([['__label__fr']], [array([0.99077034], dtype=float32)])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## gcld3.   \n",
        "\n",
        "Running the following install and import instructions might not work, because it is recommended to use a virtual environment when implementing gcld3. So I wrote the function so that you can copy-paste in your notebook/.py files."
      ],
      "metadata": {
        "id": "dlUiTFEjJ8an"
      }
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "cKRCA_90_yv5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#!pip install gcld3\n",
        "#import gcld3"
      ],
      "metadata": {
        "id": "EQ_Y5KNgT6df"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# First feature: Single Language detection\n",
        "\n",
        "def cld3_single_language_detection(text):\n",
        "\n",
        "  max_num_bytes = len(text)\n",
        "  detector = gcld3.NNetLanguageIdentifier(min_num_bytes=0, \n",
        "                                        max_num_bytes=max_num_bytes)\n",
        "  result = {}\n",
        "  result['language'] = detector.FindLanguage(text=text).language\n",
        "  result['probability'] = detector.FindLanguage(text=text).probability\n",
        "  \n",
        "  return result\n",
        "\n",
        "\n",
        "# Second feature: Multiple Language detection\n",
        "\n",
        "def cld3_multiple_language_detection(text, nb_language=2):\n",
        "    \n",
        "    max_num_bytes = len(text)\n",
        "    results = []\n",
        "    language_info = {}\n",
        "    \n",
        "    detector = gcld3.NNetLanguageIdentifier(min_num_bytes=5, \n",
        "                                        max_num_bytes=max_num_bytes)\n",
        "    \n",
        "    languages = detector.FindTopNMostFreqLangs(text=text,\n",
        "                                                 num_langs=nb_language)\n",
        "    \n",
        "    return [{\"Language\":l.language, \n",
        "             \"Probability\":l.probability} for l in languages]\n",
        "    "
      ],
      "metadata": {
        "id": "09SHATbBKCxV"
      },
      "execution_count": 33,
      "outputs": []
    }
  ]
}