SAFE_application2.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Ayfred/MissionR-D/blob/main/SAFE_application2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install jsonlines"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tu7XRMwArork",
        "outputId": "875166ae-7f02-4837-aac6-f6327bd2bf2e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: jsonlines in /usr/local/lib/python3.9/dist-packages (3.1.0)\n",
            "Requirement already satisfied: attrs>=19.2.0 in /usr/local/lib/python3.9/dist-packages (from jsonlines) (23.1.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SRP7dufeAThK",
        "outputId": "4bdb0381-a191-44ea-8a10-0366d62dddbf"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "# name = \"onlyEnComments && GooglePlayDescEN\"\n",
        "path = \"/content/drive/MyDrive/Data/onlyEnComments.csv\"\n",
        "df_content = pd.read_csv(path)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QIoLc4EtUxYi"
      },
      "source": [
        "Chargement des données CSV"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IsOHf05WUw0U"
      },
      "outputs": [],
      "source": [
        "import csv\n",
        "\n",
        "# open the CSV file using the csv module\n",
        "with open(path, newline='',  encoding='utf-8') as csvfile:\n",
        "    # create a csv reader object\n",
        "    reader = csv.reader(csvfile)\n",
        "    # create an empty list to store the strings\n",
        "    string_list = []\n",
        "    # loop through each row in the CSV file\n",
        "    for row in reader:\n",
        "        # add the string to the list\n",
        "        #if detect(row[0]) == 'fr':\n",
        "        string_list.append(row[0])"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#import jsonl & fichier annotations\n",
        "import jsonlines\n",
        "import json\n",
        "\n",
        "\n",
        "path = \"/content/drive/MyDrive/Data/AnnotedComments.jsonl\"\n",
        "annotations =[]\n",
        "\n",
        "\n",
        "with jsonlines.open(path) as reader:\n",
        "  i = 0\n",
        "  for obj in reader:\n",
        "    annotation = [i]\n",
        "    for note in obj['entities']:\n",
        "      annotation.append([note['start_offset'], note['end_offset']])\n",
        "      print(annotation)\n",
        "    if obj['entities']:\n",
        "      annotations.append(annotation)\n",
        "    i += 1\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HDyCue2PrD3Z",
        "outputId": "78522c07-bb8a-40ad-bb46-f3cfc8dcec65"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0, [25, 39]]\n",
            "[0, [25, 39], [43, 58]]\n",
            "[0, [25, 39], [43, 58], [71, 75]]\n",
            "[0, [25, 39], [43, 58], [71, 75], [80, 97]]\n",
            "[0, [25, 39], [43, 58], [71, 75], [80, 97], [160, 170]]\n",
            "[0, [25, 39], [43, 58], [71, 75], [80, 97], [160, 170], [176, 189]]\n",
            "[0, [25, 39], [43, 58], [71, 75], [80, 97], [160, 170], [176, 189], [297, 318]]\n",
            "[1, [43, 49]]\n",
            "[1, [43, 49], [53, 64]]\n",
            "[2, [19, 50]]\n",
            "[3, [100, 121]]\n",
            "[3, [100, 121], [131, 141]]\n",
            "[3, [100, 121], [131, 141], [237, 252]]\n",
            "[3, [100, 121], [131, 141], [237, 252], [260, 268]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "annotations"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QZlKP7HnuwNm",
        "outputId": "d0a692cd-9e79-4d9f-d809-ef1dd0ad807b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[[0,\n",
              "  [25, 39],\n",
              "  [43, 58],\n",
              "  [71, 75],\n",
              "  [80, 97],\n",
              "  [160, 170],\n",
              "  [176, 189],\n",
              "  [297, 318]],\n",
              " [1, [43, 49], [53, 64]],\n",
              " [2, [19, 50]],\n",
              " [3, [100, 121], [131, 141], [237, 252], [260, 268]]]"
            ]
          },
          "metadata": {},
          "execution_count": 115
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "string_list = string_list[:1000]"
      ],
      "metadata": {
        "id": "VrTWfKl3FgSS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yRWZ_PLpZCyr"
      },
      "outputs": [],
      "source": [
        "#(string_list)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r3GNJuYWTitA"
      },
      "source": [
        "Pre-traitement du texte"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AucR-PxMW9Mf",
        "outputId": "68c1c491-35c2-4c03-a0f2-e92ec14102af"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-04-19 20:04:56.074364: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting en-core-web-sm==3.5.0\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.5.0/en_core_web_sm-3.5.0-py3-none-any.whl (12.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.8/12.8 MB\u001b[0m \u001b[31m54.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: spacy<3.6.0,>=3.5.0 in /usr/local/lib/python3.9/dist-packages (from en-core-web-sm==3.5.0) (3.5.2)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (3.1.2)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.8 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (8.1.9)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.0.8)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (1.0.4)\n",
            "Requirement already satisfied: pathy>=0.10.0 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (0.10.1)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (3.3.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (4.65.0)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (1.10.7)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (3.0.12)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (3.0.8)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (23.1)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.27.1)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (6.3.0)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (1.0.9)\n",
            "Requirement already satisfied: typer<0.8.0,>=0.3.0 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (0.7.0)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (1.1.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (67.6.1)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.0.7)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.4.6)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.9/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (1.22.4)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.9/dist-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (1.26.15)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.0.12)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (3.4)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.9/dist-packages (from thinc<8.2.0,>=8.1.8->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (0.0.4)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.9/dist-packages (from thinc<8.2.0,>=8.1.8->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (0.7.9)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.9/dist-packages (from typer<0.8.0,>=0.3.0->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (8.1.3)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.9/dist-packages (from jinja2->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.1.2)\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('en_core_web_sm')\n"
          ]
        }
      ],
      "source": [
        "import re\n",
        "import spacy\n",
        "\n",
        "!python -m spacy download en_core_web_sm"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9IAx-LTiXIyb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "f9269b32-cbeb-4781-ee12-c870fca17929"
      },
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-f189ab208ae7>\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m#Etapes du texte Preprocessing\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mnlp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspacy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"en_core_web_sm\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mlistStopWords\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop_words\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'spacy' is not defined"
          ]
        }
      ],
      "source": [
        "#Etapes du texte Preprocessing\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "listStopWords = [str(x) for x in nlp.Defaults.stop_words]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "doc = nlp(string_list[0])\n",
        "for token in doc:\n",
        "  print(f\"{token.text}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "osYXzcQRzvoJ",
        "outputId": "5343263b-3844-4b7e-a334-317b613c4255"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Nicely\n",
            "designed\n",
            ",\n",
            "has\n",
            "a\n",
            "good\n",
            "selection\n",
            "of\n",
            "workout\n",
            "options\n",
            ",\n",
            "and\n",
            "does\n",
            "a\n",
            "good\n",
            "job\n",
            "tracking\n",
            "my\n",
            "sleep\n",
            "(\n",
            "via\n",
            "a\n",
            "Galaxy\n",
            "Watch\n",
            "4\n",
            ")\n",
            ".\n",
            "I\n",
            "do\n",
            "wish\n",
            "that\n",
            "there\n",
            "was\n",
            "an\n",
            "option\n",
            "to\n",
            "add\n",
            "a\n",
            "note\n",
            "or\n",
            "a\n",
            "journal\n",
            "entry\n",
            "for\n",
            "a\n",
            "sleep\n",
            "session\n",
            ",\n",
            "even\n",
            "if\n",
            "it\n",
            "was\n",
            "something\n",
            "simple\n",
            ".\n",
            "In\n",
            "regards\n",
            "to\n",
            "the\n",
            "watch\n",
            "app\n",
            ",\n",
            "I\n",
            "think\n",
            "there\n",
            "should\n",
            "be\n",
            "more\n",
            "breathing\n",
            "cycles\n",
            "in\n",
            "the\n",
            "\"\n",
            "Stress\n",
            "\"\n",
            "section\n",
            "(\n",
            "you\n",
            "can\n",
            "only\n",
            "add\n",
            "up\n",
            "to\n",
            "30\n",
            "cycles\n",
            "/\n",
            "7\n",
            "mins\n",
            ")\n",
            ".\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "K0s-fFW2TgdN"
      },
      "outputs": [],
      "source": [
        "def united(text):\n",
        "    doc =  text.split(\" \");\n",
        "    # Itérer à travers chaque token dans le document\n",
        "    for token in doc:\n",
        "        if token in [\"because\", \"since\", \"as\", ]:\n",
        "            doc = doc[:doc.index(token)]\n",
        "            break\n",
        "        #Remove explanations\n",
        "        # Si le token est \"parce que\", \"car\" ou \"puisque\", supprimer tous les tokens suivants dans la phrase\n",
        "    \n",
        "    \n",
        "    # Supprimer tout ce qui est entre parenthèses\n",
        "    sentence = ' '.join(doc)\n",
        "    #print(f\"{sentence}\")\n",
        "    sentence = re.sub(r'\\([^()]*\\)', '', sentence)\n",
        "\n",
        "    #print(f\"remove explanations : {sentence}\")\n",
        "\n",
        "    sentence = re.sub('http[s]?://\\S+', '', sentence)\n",
        "\n",
        "    #print(f\"remove links: {sentence}\")\n",
        "\n",
        "\n",
        "    # Regular expressions for phone numbers and email addresses\n",
        "    quotes = r'\\\"[^\\\"]+\\\"'\n",
        "    phone_regex = r\"(?<!\\d)(?:\\d[ -/\\\\_\\d]*){10}(?!\\d)\"\n",
        "    email_regex = r\"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}\"  # matches john.doe@example.com\n",
        "\n",
        "    # Find phone numbers and email addresses in the sentence\n",
        "    sentence = re.sub(quotes,\"\", sentence)\n",
        "\n",
        "    #print(f\"remove links2: {sentence}\")\n",
        "\n",
        "\n",
        "    sentence = re.sub(phone_regex,\"\", sentence)\n",
        "    sentence = re.sub(email_regex,\"\", sentence)\n",
        "    sentence = re.sub(r'[^\\w]', ' ', sentence)\n",
        "    doc = sentence.split(\" \")\n",
        "    for word in doc:\n",
        "        if word in listStopWords:\n",
        "            doc.remove(word)\n",
        "    \n",
        "    sentence = ' '.join(doc)\n",
        "\n",
        "    #print(f\"remove phone numbers and contacts: {sentence}\")\n",
        "\n",
        "    #Removal of ' and the char that precede it\n",
        "    sentence = sentence.replace(\"'\", \" \")\n",
        "\n",
        "    #print(f\"remove leopaul: {sentence}\")\n",
        "\n",
        "    words = sentence.split(\" \")\n",
        "    newSentence = \"\"\n",
        "    for word in words:\n",
        "        if len(word) > 1:\n",
        "            newSentence += word + \" \"\n",
        "    sentence = newSentence\n",
        "    \n",
        "    #print(f\"remove quotes : {sentence}\")\n",
        "\n",
        "    doc = nlp(sentence)\n",
        "    #texte = doc.text\n",
        "    text = \"\"\n",
        "    for token in doc:\n",
        "        if (token.pos_ == \"PRON\" or token.pos_ == \"DET\"):\n",
        "            continue\n",
        "        else:\n",
        "            if not token.text.__contains__(\"'\"):\n",
        "                text += token.text\n",
        "                text += \" \"\n",
        "    sentence = text\n",
        "\n",
        "    #print(f\"remove pronoum and determinant : {sentence}\")\n",
        "\n",
        "\n",
        "    doc = nlp(sentence)\n",
        "    result = []\n",
        "    for token in doc:\n",
        "        if not (token.ent_type_ == \"ORG\" or token.ent_type_ == \"PERSON\" or token.text.istitle()):\n",
        "            result.append(token.text)\n",
        "    sentence =  \" \".join(result)\n",
        "    #print(f\"remove proper noun : {sentence}\")\n",
        "\n",
        "    return sentence\n",
        "\n",
        "\"\"\"\n",
        "for string in string_list:\n",
        "    united_string = united(string)\n",
        "    string_list_clean.append(united_string)\n",
        "print(string_list_clean)\n",
        "#run\n",
        "#print(united(text))\"\"\"\n",
        "united_string = []\n",
        "for i in range(len(string_list)):\n",
        "  united_string += [united(string_list[i])]\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7ppmsgWCiQXk"
      },
      "outputs": [],
      "source": [
        "#united_string"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "annotation"
      ],
      "metadata": {
        "id": "A10xVcCI0urn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8b712391-13ef-4d68-d077-067b3a0422a1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1999]"
            ]
          },
          "metadata": {},
          "execution_count": 123
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i-oeT3ILTreN"
      },
      "source": [
        "Apply SAFE POS pattern"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def superpose(da, fa, db, fb):\n",
        "  return (da <= db <= fa | da <= fb <= fa | (db < da & fa < fb))\n",
        "\n",
        "\n",
        "def matchTokens(id, start, end):\n",
        "  both = 0\n",
        "  for labeled in annotations:\n",
        "    if id == labeled[0]:\n",
        "      for i in range(1, len(labeled)):\n",
        "        print(labeled[i][0], labeled[i][1], start, end)\n",
        "        if superpose(labeled[i][0], labeled[i][1], start, end):\n",
        "          both +=1\n",
        "          break\n",
        "  return both\n",
        "\n"
      ],
      "metadata": {
        "id": "cDd82HGy1478"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2lciKH5WTy0D",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8fc73992-b3eb-43cf-82f1-0313d27a72f8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "25 39 1 5\n",
            "43 58 1 5\n",
            "71 75 1 5\n",
            "80 97 1 5\n",
            "160 170 1 5\n",
            "176 189 1 5\n",
            "297 318 1 5\n",
            "25 39 6 10\n",
            "43 58 6 10\n",
            "71 75 6 10\n",
            "80 97 6 10\n",
            "160 170 6 10\n",
            "176 189 6 10\n",
            "297 318 6 10\n",
            "25 39 26 30\n",
            "43 58 26 30\n",
            "71 75 26 30\n",
            "80 97 26 30\n",
            "160 170 26 30\n",
            "176 189 26 30\n",
            "297 318 26 30\n",
            "25 39 2 6\n",
            "43 58 2 6\n",
            "71 75 2 6\n",
            "80 97 2 6\n",
            "160 170 2 6\n",
            "176 189 2 6\n",
            "297 318 2 6\n",
            "25 39 18 22\n",
            "43 58 18 22\n",
            "71 75 18 22\n",
            "80 97 18 22\n",
            "160 170 18 22\n",
            "176 189 18 22\n",
            "297 318 18 22\n",
            "25 39 14 18\n",
            "43 58 14 18\n",
            "71 75 14 18\n",
            "80 97 14 18\n",
            "160 170 14 18\n",
            "176 189 14 18\n",
            "297 318 14 18\n",
            "0\n",
            "[good selection workout options, wish option add note, , good job tracking sleep, , more breathing cycles section, accurate jnside apartment building, building, walking halls near parameter, little ways instance scheduling, , good meassure steps distance, enter data back sleep, best feature fact review, route feature, disappointing app return map, longer workout notifications steps, past week use galaxy, won load calendar longer, showing on main screen, turn phone for minute, let sign with account, wish edit time sleep, nutritional information item search, total sleep time graph, old phone monitor heart, best app fitness hands, main function app weight, inconsistent records sleep exercise, proper stretching practice idea, decent job tracking activity, reminder automatic sleep detection, automatic activity tracker sleep, able benefit data points, handy feature blood glucose, next update returns recording, non food tracking barcode, coarse exercise mispronounces words, great app health majority, great app tracking basics, graph better visual representation, years and haven issues, good graph paper grid, great app synchronizing health, easier use camera function, snap picture barcode eat, overall sleep patterns note, issue corrected, reasonable picture sleep times, , die after next charge, major issue blood glucose, great watch phone data, ReInstalled doesn work tenths, workouts dropping tenths distance, last update audio guide, open phone order snore, not having mic permissions, more OS issue app, able copy paste meals, easy understand metrics tips, was way hide section, , relevant body fitness programs, great app measuring level, great app track walks, expected lot from app, free weights exercise menu, small graph weight progress, particular hour time period, able edit sleep times, tried lot of things, last update app movements, messed period came days, , worst thing cycle tracking, asap, bad customer service fix, calirie count activities options, machine seated row, select day week selector, shows spinning icon is, current week date selector, overall data steps calorie, connect writing data point, enough features day day, using with different app, able change database food, additional power naps period, able type sleep meals, happy app step counter, similar apple health use, own step feature edit, accurate hr tracking steps, wonderful wish way track, close weekend record steps, trends so far, complete fitness challenges track, open app track way, skip login option watch, wonderful content meditation sleep, simplest items food bit, regular meals have transferred, meals plus custom meals, didn fix issue app, actual step count ratio, comparison other apps, sucks on new device, greater time spans weeks, goals thanks perks far, yearly challenge friends term, old information body composition, single troubleshooting suggestion fix, yearly step km goal, in week month aren, , total steps miles kilometers, let see totals, yearly goals steps distance, mandatory step verification glitches, latest update yesterday app, clicked on daily activity, new phone login redirects, watch pair with functionalities, options limited is way, better interface track food, fine heart rate shows, week so step counter, watch or app point, large gaps heart rate, useful tracking health targets, pressure data generate report, useful user export blood, necessary turn alarms reminders, motivational speech minimum interruption, bad none sync number, great way track health, terrible image kind exercises, worse update galaxy watch, persistent notification app running, non device health monitor, constant tracking drop alarm, plenty info health app, paper plus follow work, excellent app health fitness, try sign with account, full breathing patterns app, glad battery lasts week, stupid factor system factor, unable check stress levels, superior app track fitness, burnt measurement out sync, clear data work hour, able track sleep info, unable update weight need, unusable permit access contacts, consistent tracking steps ability, surprising failure food tracking, latest update signing error, lost results works fine, clear storage restart phone, main phone screen steps, open app devices device, tried things in attempt, type type in food, want remove from friend, work give new option, such option application health, confirmed re keen integrate, triangle and marks lines, yellow notification mark check, best health tracking app, not updated main page, ahead time discrepancy data, great fitness app track, latest update notification symbol, love app after update, app, app and oxygen sensor, multiple copies calorie entries, most time measure stress, app pick up waist, see finger on sensor, good wish change distance, ok edit hours option, wear inside wrist is, other sleep data sleep, standalone phone rate stars, great app don, listening customers of reviews, continuous heart rate tracking, wrong time duration information, day sleep occurred night, actual time records sleep, dropped update to software, accurate tracking fix issues, record bummer for night, optional permissions food library, times, missed with colorful logo, newer version stress heart, great ton workout exercises, want be considered health, tried of quick fixes, pointless use auto installing, longer stress management heart, flat boring lacks color, earlier display avg pace, working for long time, tell advise on speed, reached stamp on display, significant feature sleep cycle, simple sleep tracker fitness, able measure heart rate, fewer functions use apps, ultra measure heart rate, figured sleep of turn, able copy activity entry, correct date birth health, simple option look forums, perfect minute mile hours, weekly heart zone totals, easy check steps phone, count is changed, whole app check step, specific doesn end sleep, charging take for walk, automatic blood pressure capacity, useful app stress oxygen, frustrating sync functionality year, different provider phone tablet, keeps crashing phone have, , wished record medication app, still useless, sent hundreds of reports, daily activity rings resetting, recent update know tracking, liked on new ultra, inclined share money steal, active use app manage, able finish heart rate, correct time timezone hours, quicker tack food food, nice check blood pressure, help blood pressure blood, total seconds month sleep, same rate day fluctuations, similar issues app updates, fine use pressure feature, big snooring reports post, daily graph detail heartbit, do taxes on product, latest update step counter, old level icons wings, mobile phone women health, trustworthy fix sleep detection, morning and notice thinks, fix and retrieve data, third party apps time, , frustrated apple cares privacy, continuous heart rate graph, walk but health app, hot yoga pilates bar, actually didn grateful syncs, only option change date, great BT7000 blood pressure, add devices like speed, audio video app app, least weight recording updates, month shows 90 lb, clear error min averages, specific time reminder trigger, pausing for whole day, current update press dot, driving distances, accurate reading steps counts, achieve day from team, complete install use app, permission health app collect, valid reason grant blanket, buggy submit bug request, unable search food items, just pathetic official, educational articles nutrition exercise, cumulative info cardio suggestions, loving app about weeks, daily food track nutrition, food instance doesn track, see in nutritional breakdown, used app about years, uninstalled am looking circumstances, loved up recent changes, poor rate sleep apps, instead, interesting app use technology, daily track steps exercise, old phone update app, difficult work simplicity ease, little use reply follow, cardio option indoor walk, able measure heart rate, new phone record buying, contact first email, long customer service delays, long time user sea, able search trough category, prefixed replies like watches, fix issue next update, other fitness trackers bike, ultra work provider state, able set watch remind, installing app of options, aggressive health view details, sure time zone app, black man man bun, great way count water, intake day track calories, compare to other users, weekly summary need user, non phone login error, new phone showing server, app fault cat snores, same draw backs thing, wearable devices syncs information, issues far, easy use stabillity connection, breastfeeding moms for pregnancy, recommend to isn phone, same excuse app features, simple pixel watch face, exact route day app, unable reduce number hours, fold phone galaxy watch, fold measure button heart, recent update sleep function, s10 monitors don work, complete trash get application, new galaxy s22 section, nice copy paste workout, left of dietary equation, latest update wound uninstalling, able play music watch, draining much fast use, active watch 44 m, ultra battery saver option, new heart rate monitor, new update sleep record, deep sleep stage hour, great check heart rate, changed to older version, choose sleep from notification, wow up stairs on, whole app EXPLORE section, more ability measure stress, round circles couple years, menstrual cycle data couple, open app solve issue, past sign page sets, recent update heart rate, waste time think are, watch would been mistake, online support hour phone, now useless going buy, expensive heart rate sensor, weird email contact reply, annoying weight lifting day, available message developer response, last update sleep record, provide access to phone, difficult make option fix, new achiever level badge, going downside, bad logo user experience, great calorie counter food, intake half bar steps, big picture weight loss, real challenge ability export, previous version opinion user, total duration speed tracker, ran consecutive ever reaching, nice lots battery consumption, pretty warnings phone handle, efficient version control functions, able track averages months, feel like accurate galaxy, good tracking weight calories, nice way track step, removed obvious reason works, having of previous functionality, past months permission sync, create set of workouts, let by awful slider, great keeping track steps, overall picture ones weight, able arrange items tiles, counter bad by to, replaced by only step, own data fine eg, food etc pulse oxygenation, many glasses water sleep, huge banner app selling, deteriorated basically just tracking, additional feature friend challenge, decent step calorie counter, break in first place, old features response problem, great dev team colour, favorite feature nutrition score, had replace watch seriously, removed of best features, daily basis heart rate, removed features on trends, wrong update fix bugs, editable workout tiles home, common sense happening fitness, absolute nonsense pace values, screens displayed screen shown, related to instantaneous values, built end workout, extreme manages step count, easy use app service, works resets watch so, more steps discrepancy app, latest update ability display, circular track case speed, app press time drink, unblock phone run water]\n"
          ]
        }
      ],
      "source": [
        "from spacy.matcher import Matcher\n",
        "\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "matcher1 = Matcher(nlp.vocab)\n",
        "matcher2 = Matcher(nlp.vocab)\n",
        "matcher3 = Matcher(nlp.vocab)\n",
        "\n",
        "\"\"\"def deleteTokens(matcher, doc, i, matches):\n",
        "    # Get the current match and create tuple of entity label, start and end.\n",
        "    # Append entity to the doc's entity. (Don't overwrite doc.ents!)\n",
        "    match_id, start, end = matches[i]\n",
        "    string_id = nlp.vocab.strings[match_id]\n",
        "    text = \"\"\n",
        "\n",
        "    nlp_list = list(doc)\n",
        "    for i in range(end-start):\n",
        "        text += nlp_list[start].text + \" \"\n",
        "        del nlp_list[start]\n",
        "    print(\"\\n\")\n",
        "    print(text)\n",
        "    print(\"\\n\")\n",
        "    print(nlp_list)\n",
        "    doc = nlp(\" \".join([e.text for e in nlp_list]))\n",
        "    matched_list.append(text)\n",
        "    matched_id_list.append(string_id)\"\"\"\n",
        "\n",
        "\n",
        "# Add match ID \"HelloWorld\" with no callback and one pattern\n",
        "pattern1 = [{\"POS\": \"NOUN\"}, {\"POS\": \"NOUN\"}]\n",
        "pattern2 = [{\"POS\": \"VERB\"}, {\"POS\": \"NOUN\"}]\n",
        "pattern3 = [{\"POS\": \"ADJ\"}, {\"POS\": \"NOUN\"}]\n",
        "pattern4 = [{\"POS\": \"NOUN\"}, {\"POS\": \"CCONJ\"}, {\"POS\": \"NOUN\"}]\n",
        "pattern5 = [{\"POS\": \"ADJ\"}, {\"POS\": \"NOUN\"}, {\"POS\": \"NOUN\"}]\n",
        "pattern6 = [{\"POS\": \"NOUN\"}, {\"POS\": \"NOUN\"}, {\"POS\": \"NOUN\"}]\n",
        "pattern7 = [{\"POS\": \"VERB\"}, {\"POS\": \"PRON\"}, {\"POS\": \"NOUN\"}]\n",
        "pattern8 = [{\"POS\": \"VERB\"}, {\"POS\": \"NOUN\"}, {\"POS\": \"NOUN\"}]\n",
        "pattern9 = [{\"POS\": \"VERB\"}, {\"POS\": \"ADJ\"}, {\"POS\": \"NOUN\"}]\n",
        "pattern10 = [{\"POS\": \"ADJ\"}, {\"POS\": \"ADJ\"}, {\"POS\": \"NOUN\"}]\n",
        "pattern11 = [{\"POS\": \"NOUN\"}, {\"POS\": \"ADP\"}, {\"POS\": \"NOUN\"}]\n",
        "pattern12 = [{\"POS\": \"VERB\"}, {\"POS\": \"DET\"}, {\"POS\": \"NOUN\"}]\n",
        "pattern13 = [{\"POS\": \"VERB\"}, {\"POS\": \"NOUN\"}, {\"POS\": \"ADP\"}, {\"POS\": \"NOUN\"}]\n",
        "pattern14 = [{\"POS\": \"ADJ\"}, {\"POS\": \"NOUN\"}, {\"POS\": \"NOUN\"}, {\"POS\": \"NOUN\"}]\n",
        "pattern15 = [{\"POS\": \"ADJ\"}, {\"POS\": \"CONJ\"}, {\"POS\": \"ADJ\"}]\n",
        "pattern16 = [{\"POS\": \"VERB\"}, {\"POS\": \"ADP\"}, {\"POS\": \"ADJ\"}, {\"POS\": \"NOUN\"}]\n",
        "pattern17 = [{\"POS\": \"VERB\"}, {\"POS\": \"PRON\"}, {\"POS\": \"ADJ\"}, {\"POS\": \"NOUN\"}]\n",
        "pattern18 = [{\"POS\": \"NOUN\"}, {\"POS\": \"CCONJ\"}, {\"POS\": \"NOUN\"}, {\"POS\": \"NOUN\"}]\n",
        "matcher1.add(\"18\", [pattern18])\n",
        "matcher1.add(\"17\", [pattern17])\n",
        "matcher1.add(\"16\", [pattern16])\n",
        "matcher2.add(\"15\", [pattern15])\n",
        "matcher1.add(\"14\", [pattern14])\n",
        "matcher1.add(\"13\", [pattern13])\n",
        "matcher2.add(\"12\", [pattern12])\n",
        "matcher2.add(\"11\", [pattern11])\n",
        "matcher2.add(\"10\", [pattern10])\n",
        "matcher2.add(\"9\", [pattern9])\n",
        "matcher2.add(\"8\", [pattern8])\n",
        "matcher2.add(\"7\", [pattern7])\n",
        "matcher2.add(\"6\", [pattern6])\n",
        "matcher2.add(\"5\", [pattern5])\n",
        "matcher2.add(\"4\", [pattern4])\n",
        "matcher3.add(\"3\", [pattern3])\n",
        "matcher3.add(\"2\", [pattern2])\n",
        "matcher3.add(\"1\", [pattern1])\n",
        "\n",
        "matched_id_list = []\n",
        "matched_list = []\n",
        "#doc = nlp(\"Song and artist album,  Create your greatest album, \")\n",
        "\n",
        "for row in range(len(united_string)):\n",
        "  doc = nlp(united_string[row])\n",
        "  while(True):\n",
        "    matches = matcher1(doc)\n",
        "    for match_id, start, end in matches:\n",
        "      trupos = matchTokens(row, start, end)\n",
        "      string_id = nlp.vocab.strings[match_id]\n",
        "      nlp_list = list(doc)\n",
        "      #print(doc[start:end])\n",
        "      matched_list.append(doc[start:end])\n",
        "      matched_id_list.append(string_id)\n",
        "\n",
        "      for i in range(end-start):\n",
        "        if(start<len(nlp_list)):\n",
        "          del nlp_list[start]\n",
        "\n",
        "      doc = nlp(\" \".join([e.text for e in nlp_list]))\n",
        "    if (not matches):\n",
        "      break\n",
        "  while(True):\n",
        "    matches = matcher1(doc)\n",
        "    #trupos += matchTokens(row, matches)\n",
        "    for match_id, start, end in matches:\n",
        "      string_id = nlp.vocab.strings[match_id]\n",
        "      nlp_list = list(doc)\n",
        "      print(doc[start:end])\n",
        "      matched_list.append(doc[start:end])\n",
        "      matched_id_list.append(string_id)\n",
        "\n",
        "      for i in range(end-start):\n",
        "        if(start<len(nlp_list)):\n",
        "          del nlp_list[start]\n",
        "\n",
        "      doc = nlp(\" \".join([e.text for e in nlp_list]))\n",
        "    if (not matches):\n",
        "      break\n",
        "  while(True):\n",
        "    matches = matcher1(doc)\n",
        "    #trupos += matchTokens(row, matches)\n",
        "    for match_id, start, end in matches:\n",
        "      string_id = nlp.vocab.strings[match_id]\n",
        "      nlp_list = list(doc)\n",
        "      print(doc[start:end])\n",
        "      matched_list.append(doc[start:end])\n",
        "      matched_id_list.append(string_id)\n",
        "\n",
        "      for i in range(end-start):\n",
        "        if(start<len(nlp_list)):\n",
        "          del nlp_list[start]\n",
        "\n",
        "      doc = nlp(\" \".join([e.text for e in nlp_list]))\n",
        "    if (not matches):\n",
        "      break\n",
        "\n",
        "print(trupos)\n",
        "#Remove duplicate and synonyms, noise\n",
        "print(matched_list)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RHiRNh4gr2pK"
      },
      "outputs": [],
      "source": [
        "for el in matched_list:\n",
        "  if len(el.text.split(\" \"))<=1:\n",
        "    matched_list.remove(el)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(matched_list)"
      ],
      "metadata": {
        "id": "tybcG713PY54"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(matched_list)"
      ],
      "metadata": {
        "id": "ke9yEphpob6S"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "o4jbgN19tDpI"
      },
      "outputs": [],
      "source": [
        "matched_list"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -U sentence-transformers\n",
        "from sentence_transformers import SentenceTransformer\n",
        "model = SentenceTransformer('all-MiniLM-L6-v2')"
      ],
      "metadata": {
        "id": "pY456lm7Maow"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install faiss-cpu\n",
        "import faiss\n",
        "from sklearn.metrics import silhouette_score\n",
        "import numpy as np\n",
        "from sentence_transformers import SentenceTransformer\n",
        "\n",
        "# Load pre-trained language model\n",
        "model = SentenceTransformer('all-MiniLM-L6-v2')\n",
        "\n",
        "# Define number of clusters\n",
        "num_clusters = 63\n",
        "\n",
        "# Define the list of phrases to cluster\n",
        "phrases = matched_list\n",
        "\n",
        "# Encode the phrases using the pre-trained model\n",
        "phrase_embeddings = model.encode(phrases)\n",
        "\n",
        "# Cluster the embeddings using K-means\n",
        "from sklearn.cluster import KMeans\n",
        "kmeans = KMeans(n_clusters=num_clusters, random_state=0).fit(phrase_embeddings)\n",
        "\n",
        "# Compute the silhouette score for the clustering\n",
        "silhouette_avg = silhouette_score(phrase_embeddings, kmeans.labels_)\n",
        "\n",
        "print(\"The average silhouette_score is :\", silhouette_avg)\n",
        "\n",
        "# Assign each phrase to a cluster\n",
        "clusters = [[] for i in range(num_clusters)]\n",
        "for i, label in enumerate(kmeans.labels_):\n",
        "    clusters[label].append(phrases[i])\n",
        "\n",
        "# Print the clusters\n",
        "for i in range(num_clusters):\n",
        "    print(\"Cluster {}:\".format(i+1))\n",
        "    print(clusters[i])\n"
      ],
      "metadata": {
        "id": "ouXR8d26aMi6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sentence_transformers import SentenceTransformer\n",
        "from sklearn.cluster import DBSCAN\n",
        "from sklearn.metrics import silhouette_score\n",
        "import numpy as np\n",
        "import faiss\n",
        "\n",
        "embedder = SentenceTransformer('all-MiniLM-L6-v2')\n",
        "\n",
        "# Corpus with example sentences\n",
        "corpus = matched_list\n",
        "corpus_embeddings = embedder.encode(corpus)\n",
        "\n",
        "# Normalize the embeddings to unit length\n",
        "corpus_embeddings = corpus_embeddings / np.linalg.norm(corpus_embeddings, axis=1, keepdims=True)\n",
        "\n",
        "# Set up Faiss for nearest neighbor search\n",
        "index = faiss.IndexFlatIP(corpus_embeddings.shape[1])\n",
        "index.add(corpus_embeddings)\n",
        "\n",
        "# Perform nearest neighbor search to get indices of similar embeddings\n",
        "_, indices = index.search(corpus_embeddings, k=5)  # Get the 5 most similar embeddings for each sentence\n",
        "\n",
        "# Perform DBSCAN clustering on the similar embeddings\n",
        "clustering_model = DBSCAN(eps=0.42, min_samples=5, metric='cosine')\n",
        "\n",
        "# Perform DBSCAN clustering on the similar embeddings\n",
        "clustering_model = DBSCAN(eps=0.42, min_samples=5, metric='euclidean')\n",
        "\n",
        "# Perform HDBSCAN clustering on the similar embeddings\n",
        "clustering_model = hdbscan.HDBSCAN(min_cluster_size=5, metric='euclidean', cluster_selection_epsilon=0.40)\n",
        "\n",
        "\n",
        "cluster_assignment = clustering_model.fit_predict(corpus_embeddings[indices[:, 1]])\n",
        "\n",
        "# Compute the silhouette score for the clustering\n",
        "silhouette_avg = silhouette_score(corpus_embeddings[indices[:, 1]], cluster_assignment)\n",
        "\n",
        "print(\"The average silhouette_score is :\", silhouette_avg)\n",
        "\n",
        "clustered_sentences = {}\n",
        "for sentence_id, cluster_id in enumerate(cluster_assignment):\n",
        "    if cluster_id not in clustered_sentences:\n",
        "        clustered_sentences[cluster_id] = []\n",
        "\n",
        "    clustered_sentences[cluster_id].append(corpus[sentence_id])\n",
        "\n",
        "for i, cluster in clustered_sentences.items():\n",
        "    print(\"Cluster \", i+1)\n",
        "    print(cluster)\n",
        "    print(\"\")\n"
      ],
      "metadata": {
        "id": "qMmbWbtxa9fJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\"\"\"\n",
        "This is a simple application for sentence embeddings: clustering\n",
        "Sentences are mapped to sentence embeddings and then k-mean clustering is applied.\n",
        "\"\"\"\n",
        "from sentence_transformers import SentenceTransformer\n",
        "from sklearn.cluster import KMeans\n",
        "\n",
        "embedder = SentenceTransformer('all-mpnet-base-v2')\n",
        "\n",
        "# Corpus with example sentences\n",
        "corpus = matched_list\n",
        "corpus_embeddings = embedder.encode(corpus)\n",
        "\n",
        "# Perform kmean clustering\n",
        "num_clusters = 63\n",
        "clustering_model = KMeans(n_clusters=num_clusters)\n",
        "clustering_model.fit(corpus_embeddings)\n",
        "cluster_assignment = clustering_model.labels_\n",
        "\n",
        "clustered_sentences = [[] for i in range(num_clusters)]\n",
        "for sentence_id, cluster_id in enumerate(cluster_assignment):\n",
        "    clustered_sentences[cluster_id].append(corpus[sentence_id])\n",
        "\n",
        "for i, cluster in enumerate(clustered_sentences):\n",
        "    print(\"Cluster \", i+1)\n",
        "    print(cluster)\n",
        "    print(\"\")"
      ],
      "metadata": {
        "id": "BCm_99BgkdDa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\"\"\"\n",
        "This is a simple application for sentence embeddings: clustering\n",
        "\n",
        "Sentences are mapped to sentence embeddings and then agglomerative clustering with a threshold is applied.\n",
        "\"\"\"\n",
        "from sentence_transformers import SentenceTransformer\n",
        "from sklearn.cluster import AgglomerativeClustering\n",
        "import numpy as np\n",
        "\n",
        "embedder = SentenceTransformer('all-MiniLM-L6-v2')\n",
        "\n",
        "# Corpus with example sentences\n",
        "corpus = matched_list\n",
        "corpus_embeddings = embedder.encode(corpus)\n",
        "\n",
        "# Normalize the embeddings to unit length\n",
        "corpus_embeddings = corpus_embeddings /  np.linalg.norm(corpus_embeddings, axis=1, keepdims=True)\n",
        "\n",
        "# Perform kmean clustering\n",
        "clustering_model = AgglomerativeClustering(n_clusters=None, distance_threshold=1.5) #, affinity='cosine', linkage='average', distance_threshold=0.4)\n",
        "clustering_model.fit(corpus_embeddings)\n",
        "cluster_assignment = clustering_model.labels_\n",
        "\n",
        "# Compute the silhouette score for the clustering\n",
        "silhouette_avg = silhouette_score(corpus_embeddings, cluster_assignment)\n",
        "\n",
        "print(\"The average silhouette_score is :\", silhouette_avg)\n",
        "\n",
        "clustered_sentences = {}\n",
        "for sentence_id, cluster_id in enumerate(cluster_assignment):\n",
        "    if cluster_id not in clustered_sentences:\n",
        "        clustered_sentences[cluster_id] = []\n",
        "\n",
        "    clustered_sentences[cluster_id].append(corpus[sentence_id])\n",
        "\n",
        "for i, cluster in clustered_sentences.items():\n",
        "    print(\"Cluster \", i+1)\n",
        "    print(cluster)\n",
        "    print(\"\")"
      ],
      "metadata": {
        "id": "z_O88qI8lBuz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sentence_transformers import SentenceTransformer\n",
        "from sklearn.cluster import DBSCAN\n",
        "import numpy as np\n",
        "\n",
        "embedder = SentenceTransformer('all-mpnet-base-v2')\n",
        "\n",
        "# Corpus with example sentences\n",
        "corpus = matched_list\n",
        "corpus_embeddings = embedder.encode(corpus)\n",
        "\n",
        "# Normalize the embeddings to unit length\n",
        "corpus_embeddings = corpus_embeddings /  np.linalg.norm(corpus_embeddings, axis=1, keepdims=True)\n",
        "\n",
        "# Perform DBSCAN clustering\n",
        "clustering_model = DBSCAN(eps=0.5, min_samples=2, metric='cosine')\n",
        "cluster_assignment = clustering_model.fit_predict(corpus_embeddings)\n",
        "\n",
        "# Compute the silhouette score for the clustering\n",
        "silhouette_avg = silhouette_score(corpus_embeddings[indices[:, 1]], cluster_assignment)\n",
        "\n",
        "print(silhouette_avg)\n",
        "\n",
        "clustered_sentences = {}\n",
        "for sentence_id, cluster_id in enumerate(cluster_assignment):\n",
        "    if cluster_id not in clustered_sentences:\n",
        "        clustered_sentences[cluster_id] = []\n",
        "\n",
        "    clustered_sentences[cluster_id].append(corpus[sentence_id])\n",
        "\n",
        "for i, cluster in clustered_sentences.items():\n",
        "    print(\"Cluster \", i+1)\n",
        "    print(cluster)\n",
        "    print(\"\")\n"
      ],
      "metadata": {
        "id": "tx7Ad_ESoj10"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sentence_transformers import SentenceTransformer\n",
        "from sklearn.cluster import DBSCAN\n",
        "import numpy as np\n",
        "import faiss\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.decomposition import PCA\n",
        "\n",
        "embedder = SentenceTransformer('all-MiniLM-L6-v2')\n",
        "\n",
        "# Corpus with example sentences\n",
        "corpus = matched_list\n",
        "corpus_embeddings = embedder.encode(corpus)\n",
        "\n",
        "# Normalize the embeddings to unit length\n",
        "corpus_embeddings = corpus_embeddings / np.linalg.norm(corpus_embeddings, axis=1, keepdims=True)\n",
        "\n",
        "# Set up Faiss for nearest neighbor search\n",
        "index = faiss.IndexFlatIP(corpus_embeddings.shape[1])\n",
        "index.add(corpus_embeddings)\n",
        "\n",
        "# Perform nearest neighbor search to get indices of similar embeddings\n",
        "_, indices = index.search(corpus_embeddings, k=5)  # Get the 5 most similar embeddings for each sentence\n",
        "\n",
        "# Perform DBSCAN clustering on the similar embeddings\n",
        "clustering_model = DBSCAN(eps=1.5, min_samples=2, metric='cosine')\n",
        "cluster_assignment = clustering_model.fit_predict(corpus_embeddings[indices[:, 1]])\n",
        "\n",
        "\n",
        "# Reduce dimensionality for visualization\n",
        "pca = PCA(n_components=2)\n",
        "corpus_embeddings_2d = pca.fit_transform(corpus_embeddings)\n",
        "\n",
        "# Plot the clustered embeddings\n",
        "fig, ax = plt.subplots()\n",
        "scatter = ax.scatter(corpus_embeddings_2d[:, 0], corpus_embeddings_2d[:, 1], c=cluster_assignment)\n",
        "legend1 = ax.legend(*scatter.legend_elements(),\n",
        "                    loc=\"upper right\", title=\"Clusters\")\n",
        "ax.add_artist(legend1)\n",
        "plt.show()\n"
      ],
      "metadata": {
        "id": "RaQ4v7GdsQx4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install hdbscan"
      ],
      "metadata": {
        "id": "f7Pnv5WEbLEm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sentence_transformers import SentenceTransformer\n",
        "import numpy as np\n",
        "import faiss\n",
        "import hdbscan\n",
        "from sklearn.metrics import silhouette_score\n",
        "\n",
        "embedder = SentenceTransformer('all-MiniLM-L6-v2')\n",
        "\n",
        "# Corpus with example sentences\n",
        "corpus = matched_list\n",
        "corpus_embeddings = embedder.encode(corpus)\n",
        "\n",
        "# Normalize the embeddings to unit length\n",
        "corpus_embeddings = corpus_embeddings / np.linalg.norm(corpus_embeddings, axis=1, keepdims=True)\n",
        "\n",
        "# Set up Faiss for nearest neighbor search\n",
        "index = faiss.IndexFlatIP(corpus_embeddings.shape[1])\n",
        "index.add(corpus_embeddings)\n",
        "\n",
        "# Perform nearest neighbor search to get indices of similar embeddings\n",
        "_, indices = index.search(corpus_embeddings, k=5)  # Get the 5 most similar embeddings for each sentence\n",
        "\n",
        "# Perform HDBSCAN clustering on the similar embeddings\n",
        "clustering_model = hdbscan.HDBSCAN(min_cluster_size=5, metric='euclidean', cluster_selection_epsilon=0.40)\n",
        "cluster_assignment = clustering_model.fit_predict(corpus_embeddings[indices[:, 1]])\n",
        "\n",
        "# Compute the silhouette score for the clustering\n",
        "silhouette_avg = silhouette_score(corpus_embeddings[indices[:, 1]], cluster_assignment)\n",
        "\n",
        "print(\"The average silhouette_score is :\", silhouette_avg)\n",
        "\n",
        "from sklearn.metrics import calinski_harabasz_score\n",
        "\n",
        "ch_score = calinski_harabasz_score(corpus_embeddings[indices[:, 1]], cluster_assignment)\n",
        "\n",
        "print(\"The Calinski-Harabasz score is :\", ch_score)\n",
        "\n",
        "from sklearn.metrics import davies_bouldin_score\n",
        "\n",
        "db_score = davies_bouldin_score(corpus_embeddings[indices[:, 1]], cluster_assignment)\n",
        "\n",
        "print(\"The Davies-Bouldin score is :\", db_score)\n",
        "\n",
        "\n",
        "\n",
        "clustered_sentences = {}\n",
        "for sentence_id, cluster_id in enumerate(cluster_assignment):\n",
        "    if cluster_id not in clustered_sentences:\n",
        "        clustered_sentences[cluster_id] = []\n",
        "\n",
        "    clustered_sentences[cluster_id].append(corpus[sentence_id])\n",
        "\n",
        "for i, cluster in clustered_sentences.items():\n",
        "    print(\"Cluster \", i+1)\n",
        "    print(cluster)\n",
        "    print(\"\")\n"
      ],
      "metadata": {
        "id": "X2-71x8UbJNg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.decomposition import LatentDirichletAllocation\n",
        "import matplotlib.pyplot as plt\n",
        "import networkx as nx\n",
        "\n",
        "# List of features\n",
        "features = matched_list\n",
        "\n",
        "# Preprocess the features\n",
        "# You can use NLTK or spaCy for preprocessing\n",
        "preprocessed_features = [feat.lower().split() for feat in features]\n",
        "\n",
        "# Create a document-term matrix\n",
        "vectorizer = CountVectorizer()\n",
        "dtm = vectorizer.fit_transform([' '.join(feat) for feat in preprocessed_features])\n",
        "\n",
        "# Identify topics using LDA\n",
        "lda = LatentDirichletAllocation(n_components=3, random_state=0)\n",
        "doc_topic_dist = lda.fit_transform(dtm)\n",
        "\n",
        "# Create a topic graph\n",
        "G = nx.Graph()\n",
        "for i, feat in enumerate(features):\n",
        "    topic = doc_topic_dist[i].argmax()\n",
        "    G.add_node(feat, topic=topic)\n",
        "\n",
        "# Add edges between nodes with the same topic\n",
        "for u, v in G.edges():\n",
        "    if G.nodes[u]['topic'] == G.nodes[v]['topic']:\n",
        "        G.add_edge(u, v)\n",
        "\n",
        "# Draw the topic graph\n",
        "pos = nx.spring_layout(G, k=0.5, iterations=50)\n",
        "colors = ['red', 'green', 'blue']\n",
        "for topic in range(3):\n",
        "    nodes = [node for node in G.nodes() if G.nodes[node]['topic'] == topic]\n",
        "    nx.draw_networkx_nodes(G, pos, nodelist=nodes, node_color=colors[topic], alpha=0.8)\n",
        "nx.draw_networkx_edges(G, pos, edge_color='gray', alpha=0.5)\n",
        "nx.draw_networkx_labels(G, pos, font_size=10, font_family='sans-serif')\n",
        "plt.axis('off')\n",
        "plt.show()\n"
      ],
      "metadata": {
        "id": "tGzbWTLcdyHQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install gensim"
      ],
      "metadata": {
        "id": "8M8rEf6eybUE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sentence_transformers import SentenceTransformer\n",
        "from sklearn.cluster import DBSCAN\n",
        "import numpy as np\n",
        "import faiss\n",
        "#from gensim.summarization import keywords\n",
        "\n",
        "embedder = SentenceTransformer('all-MiniLM-L6-v2')\n",
        "\n",
        "# Corpus with example sentences\n",
        "corpus = matched_list\n",
        "corpus_embeddings = embedder.encode(corpus)\n",
        "\n",
        "# Normalize the embeddings to unit length\n",
        "corpus_embeddings = corpus_embeddings / np.linalg.norm(corpus_embeddings, axis=1, keepdims=True)\n",
        "\n",
        "# Set up Faiss for nearest neighbor search\n",
        "index = faiss.IndexFlatIP(corpus_embeddings.shape[1])\n",
        "index.add(corpus_embeddings)\n",
        "\n",
        "# Perform nearest neighbor search to get indices of similar embeddings\n",
        "_, indices = index.search(corpus_embeddings, k=5)  # Get the 5 most similar embeddings for each sentence\n",
        "\n",
        "# Perform DBSCAN clustering on the similar embeddings\n",
        "clustering_model = DBSCAN(eps=1.5, min_samples=2, metric='euclidean')\n",
        "\n",
        "\n",
        "cluster_assignment = clustering_model.fit_predict(corpus_embeddings[indices[:, 1]])\n",
        "\n",
        "# Automatically extract keywords from sentences in each cluster and use them as cluster names\n",
        "clustered_sentences = {}\n",
        "for sentence_id, cluster_id in enumerate(cluster_assignment):\n",
        "    if cluster_id not in clustered_sentences:\n",
        "        clustered_sentences[cluster_id] = []\n",
        "\n",
        "    clustered_sentences[cluster_id].append(corpus[sentence_id])\n",
        "\n",
        "cluster_names = {}\n",
        "for cluster_id, sentences in clustered_sentences.items():\n",
        "    keywords_str = keywords('\\n'.join(sentences))\n",
        "    cluster_names[cluster_id] = keywords_str.split('\\n')[0] if keywords_str else f\"Cluster {cluster_id+1}\"\n",
        "\n",
        "# Print clusters with their corresponding names\n",
        "for cluster_id, sentences in clustered_sentences.items():\n",
        "    print(f\"Cluster {cluster_id+1} - {cluster_names[cluster_id]}\")\n",
        "    print(sentences)\n",
        "    print(\"\")\n"
      ],
      "metadata": {
        "id": "YAkpqMnVyYTq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "B9evoOemLW6u"
      },
      "outputs": [],
      "source": [
        "from sentence_transformers import SentenceTransformer, util\n",
        "import torch\n",
        "\n",
        "embedder = SentenceTransformer('all-MiniLM-L6-v2')\n",
        "\n",
        "# Corpus with example sentences\n",
        "corpus = matched_list[1:]\n",
        "corpus_embeddings = embedder.encode(corpus, convert_to_tensor=True)\n",
        "\n",
        "# Query sentences:\n",
        "queries = [matched_list[0]]\n",
        "\n",
        "for queries in matched_list:\n",
        "  # Find the closest 5 sentences of the corpus for each query sentence based on cosine similarity\n",
        "  top_k = min(100, len(corpus))\n",
        "  for query in queries:\n",
        "    query_embedding = embedder.encode(query, convert_to_tensor=True)\n",
        "\n",
        "    # We use cosine-similarity and torch.topk to find the highest 5 scores\n",
        "    cos_scores = util.cos_sim(query_embedding, corpus_embeddings)[0]\n",
        "    top_results = torch.topk(cos_scores, k=top_k)\n",
        "\n",
        "    for score, idx in zip(top_results[0], top_results[1]):\n",
        "        if(score > 0,6):\n",
        "          matched_list.pop(idx)\n",
        "      \n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install faiss-cpu"
      ],
      "metadata": {
        "id": "3z2YiPbAog0B"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import faiss\n",
        "import numpy as np\n",
        "from sentence_transformers import SentenceTransformer, util\n",
        "\n",
        "#On charge le modèle de transformation de phrases \"all-MiniLM-L6-v2\" à partir de la bibliothèque SentenceTransformer.\n",
        "\n",
        "model = SentenceTransformer('all-MiniLM-L6-v2')\n",
        "\n",
        "#le nombre de voisins les plus proches à récupérer.\n",
        "# Define the number of nearest neighbors to retrieve\n",
        "k = 5\n",
        "\n",
        "# Define the corpus of sentences\n",
        "corpus = matched_list[1:]\n",
        "\n",
        "# Encoder le corpus de phrases en utilisant le modèle de transformation de phrases.\n",
        "corpus_embeddings = model.encode(corpus)\n",
        "\n",
        "# Embed the query sentence\n",
        "query = matched_list[0]\n",
        "query_embedding = model.encode([query])\n",
        "\n",
        "# Create an index for the corpus embeddings\n",
        "#On crée un index FAISS pour les embeddings des phrases dans le corpus,\n",
        "#en utilisant l'algorithme \"IndexFlatIP\" pour calculer le produit scalaire entre les embeddings.\n",
        "d = len(corpus_embeddings[0])\n",
        "index = faiss.IndexFlatIP(d)\n",
        "index.add(np.array(corpus_embeddings))\n",
        "\n",
        "# Search for the k nearest neighbors\n",
        "D, I = index.search(np.array(query_embedding), k)\n",
        "\n",
        "# Print the results\n",
        "print(\"\\n\\n======================\\n\\n\")\n",
        "print(\"Query:\", query)\n",
        "print(\"\\nTop 5 most similar sentences in corpus:\")\n",
        "for i, idx in enumerate(I[0]):\n",
        "    print(corpus[idx], \"(Score: {:.4f})\".format(D[0][i]))"
      ],
      "metadata": {
        "id": "0hE0U9VkoqP9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import faiss\n",
        "import numpy as np\n",
        "from sentence_transformers import SentenceTransformer, util\n",
        "from sklearn.cluster import DBSCAN\n",
        "\n",
        "#On charge le modèle de transformation de phrases \"all-MiniLM-L6-v2\" à partir de la bibliothèque SentenceTransformer.\n",
        "\n",
        "model = SentenceTransformer('all-MiniLM-L6-v2')\n",
        "\n",
        "#le nombre de voisins les plus proches à récupérer.\n",
        "# Define the number of nearest neighbors to retrieve\n",
        "k = 5\n",
        "\n",
        "# Define the corpus of sentences\n",
        "corpus = matched_list[1:]\n",
        "\n",
        "# Encoder le corpus de phrases en utilisant le modèle de transformation de phrases.\n",
        "corpus_embeddings = model.encode(corpus)\n",
        "\n",
        "# Embed the query sentence\n",
        "query = matched_list[0]\n",
        "query_embedding = model.encode([query])\n",
        "\n",
        "# Create an index for the corpus embeddings\n",
        "#On crée un index FAISS pour les embeddings des phrases dans le corpus,\n",
        "#en utilisant l'algorithme \"IndexFlatIP\" pour calculer le produit scalaire entre les embeddings.\n",
        "d = len(corpus_embeddings[0])\n",
        "index = faiss.IndexFlatIP(d)\n",
        "index.add(np.array(corpus_embeddings))\n",
        "\n",
        "# Search for the k nearest neighbors\n",
        "D, I = index.search(np.array(query_embedding), k)\n",
        "\n",
        "# Get the features to be clustered\n",
        "features = np.array(matched_list)\n",
        "\n",
        "# Define the parameters for the DBSCAN clustering algorithm\n",
        "eps = 0.5\n",
        "min_samples = 5\n",
        "\n",
        "# Instantiate a DBSCAN object with the defined parameters\n",
        "dbscan = DBSCAN(eps=eps, min_samples=min_samples)\n",
        "\n",
        "# Fit the DBSCAN object to the features\n",
        "dbscan.fit(features)\n",
        "\n",
        "# Get the labels for each point in the features\n",
        "labels = dbscan.labels_\n",
        "\n",
        "# Get the unique labels (i.e., cluster IDs) in the labels array\n",
        "unique_labels = np.unique(labels)\n",
        "\n",
        "# Loop through each label to print the corresponding features\n",
        "for label in unique_labels:\n",
        "    if label == -1:\n",
        "        # This is noise, i.e., a feature that doesn't belong to any cluster\n",
        "        print(\"Noise:\")\n",
        "    else:\n",
        "        # This is a cluster\n",
        "        print(\"Cluster\", label, \":\")\n",
        "    # Print the features in the current cluster\n",
        "    for idx, feature in enumerate(features[labels == label]):\n",
        "        print(\"\\t\", idx+1, \":\", feature)"
      ],
      "metadata": {
        "id": "MMR0KtF7XjQG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sentence_transformers import SentenceTransformer\n",
        "import numpy as np\n",
        "import faiss\n",
        "import hdbscan\n",
        "from sklearn.metrics import silhouette_score\n",
        "\n",
        "embedder = SentenceTransformer('all-MiniLM-L6-v2')\n",
        "\n",
        "# Corpus with example sentences\n",
        "corpus = matched_list\n",
        "corpus_embeddings = embedder.encode(corpus)\n",
        "\n",
        "# Normalize the embeddings to unit length\n",
        "corpus_embeddings = corpus_embeddings / np.linalg.norm(corpus_embeddings, axis=1, keepdims=True)\n",
        "\n",
        "# Set up Faiss for nearest neighbor search\n",
        "index = faiss.IndexFlatIP(corpus_embeddings.shape[1])\n",
        "index.add(corpus_embeddings)\n",
        "\n",
        "# Perform nearest neighbor search to get indices of similar embeddings\n",
        "_, indices = index.search(corpus_embeddings, k=5)  # Get the 5 most similar embeddings for each sentence\n",
        "\n",
        "# Perform HDBSCAN clustering on the similar embeddings\n",
        "clustering_model = hdbscan.HDBSCAN(min_cluster_size=5, metric='euclidean', cluster_selection_epsilon=0.40)\n",
        "cluster_assignment = clustering_model.fit_predict(corpus_embeddings[indices[:, 1]])\n",
        "\n",
        "# Compute the silhouette score for the clustering\n",
        "silhouette_avg = silhouette_score(corpus_embeddings[indices[:, 1]], cluster_assignment)\n",
        "\n",
        "print(\"The average silhouette_score is :\", silhouette_avg)\n",
        "\n",
        "from sklearn.metrics import calinski_harabasz_score\n",
        "\n",
        "ch_score = calinski_harabasz_score(corpus_embeddings[indices[:, 1]], cluster_assignment)\n",
        "\n",
        "print(\"The Calinski-Harabasz score is :\", ch_score)\n",
        "\n",
        "from sklearn.metrics import davies_bouldin_score\n",
        "\n",
        "db_score = davies_bouldin_score(corpus_embeddings[indices[:, 1]], cluster_assignment)\n",
        "\n",
        "print(\"The Davies-Bouldin score is :\", db_score)\n",
        "\n",
        "\n",
        "\n",
        "clustered_sentences = {}\n",
        "for sentence_id, cluster_id in enumerate(cluster_assignment):\n",
        "    if cluster_id not in clustered_sentences:\n",
        "        clustered_sentences[cluster_id] = []\n",
        "\n",
        "    clustered_sentences[cluster_id].append(corpus[sentence_id])\n",
        "\n",
        "for i, cluster in clustered_sentences.items():\n",
        "    print(\"Cluster \", i+1)\n",
        "    print(cluster)\n",
        "    print(\"\")\n"
      ],
      "metadata": {
        "id": "wKMu4ESEYWlz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "matched_list_str = [str(k) for k in matched_list]"
      ],
      "metadata": {
        "id": "4g2ZbFDOa3mf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.decomposition import LatentDirichletAllocation\n",
        "import matplotlib.pyplot as plt\n",
        "import networkx as nx\n",
        "\n",
        "# List of features\n",
        "features = matched_list_str # Your list of 1000+ features\n",
        "\n",
        "# Preprocess the features\n",
        "# You can use NLTK or spaCy for preprocessing\n",
        "preprocessed_features = [feat.lower().split() for feat in features]\n",
        "\n",
        "# Create a document-term matrix\n",
        "vectorizer = CountVectorizer()\n",
        "dtm = vectorizer.fit_transform([' '.join(feat) for feat in preprocessed_features])\n",
        "\n",
        "# Identify topics using LDA\n",
        "lda = LatentDirichletAllocation(n_components=30, random_state=0)\n",
        "doc_topic_dist = lda.fit_transform(dtm)\n",
        "\n",
        "# Create a topic graph\n",
        "G = nx.Graph()\n",
        "for i, feat in enumerate(features):\n",
        "    topic = doc_topic_dist[i].argmax()\n",
        "    G.add_node(feat, topic=topic)\n",
        "\n",
        "# Add edges between nodes with the same topic\n",
        "for u, v in G.edges():\n",
        "    if G.nodes[u]['topic'] == G.nodes[v]['topic']:\n",
        "        G.add_edge(u, v)\n",
        "\n",
        "# Draw the topic graph\n",
        "pos = nx.spring_layout(G, k=0.5, iterations=50)\n",
        "colors = ['red', 'green', 'blue']\n",
        "for topic in range(3):\n",
        "    nodes = [node for node in G.nodes() if G.nodes[node]['topic'] == topic]\n",
        "    nx.draw_networkx_nodes(G, pos, nodelist=nodes, node_color=colors[topic], alpha=0.8)\n",
        "nx.draw_networkx_edges(G, pos, edge_color='gray', alpha=0.5)\n",
        "nx.draw_networkx_labels(G, pos, font_size=10, font_family='sans-serif')\n",
        "plt.axis('off')\n",
        "plt.figure(figsize=(20, 15))\n",
        "plt.show()\n"
      ],
      "metadata": {
        "id": "A7-ZYud1a6HQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.decomposition import LatentDirichletAllocation\n",
        "import matplotlib.pyplot as plt\n",
        "import networkx as nx\n",
        "\n",
        "# List of features\n",
        "features = matched_list_str[:100]  # Your list of 1000+ features\n",
        "\n",
        "# Preprocess the features\n",
        "# You can use NLTK or spaCy for preprocessing\n",
        "preprocessed_features = [feat.lower().split() for feat in features]\n",
        "\n",
        "# Create a document-term matrix\n",
        "vectorizer = CountVectorizer()\n",
        "dtm = vectorizer.fit_transform([' '.join(feat) for feat in preprocessed_features])\n",
        "\n",
        "# Identify topics using LDA\n",
        "n_topics = 1\n",
        "lda = LatentDirichletAllocation(n_components=n_topics, random_state=0)\n",
        "doc_topic_dist = lda.fit_transform(dtm)\n",
        "\n",
        "# Create a topic graph\n",
        "G = nx.Graph()\n",
        "for i, feat in enumerate(features):\n",
        "    topic = doc_topic_dist[i].argmax()\n",
        "    G.add_node(feat, topic=topic, prob=doc_topic_dist[i][topic])\n",
        "\n",
        "# Add edges between nodes with the same topic\n",
        "for u, v in G.edges():\n",
        "    if G.nodes[u]['topic'] == G.nodes[v]['topic']:\n",
        "        G.add_edge(u, v)\n",
        "\n",
        "# Filter out features with low topic probability\n",
        "top_n = 100\n",
        "for topic in range(n_topics):\n",
        "    nodes = [node for node in G.nodes() if G.nodes[node]['topic'] == topic]\n",
        "    nodes_sorted = sorted(nodes, key=lambda x: G.nodes[x]['prob'], reverse=True)\n",
        "    nodes_top = nodes_sorted[:top_n]\n",
        "    for node in nodes_top:\n",
        "        G.nodes[node]['label'] = node.split()[-1]\n",
        "\n",
        "# Draw the topic graph\n",
        "pos = nx.spring_layout(G, k=0.5, iterations=50)\n",
        "colors = plt.cm.get_cmap('hsv', n_topics)\n",
        "for topic in range(n_topics):\n",
        "    nodes = [node for node in G.nodes() if G.nodes[node]['topic'] == topic]\n",
        "    nodes_top = [node for node in nodes if 'label' in G.nodes[node]]\n",
        "    node_colors = [colors(topic) for _ in range(len(nodes_top))]\n",
        "    nx.draw_networkx_nodes(G, pos, nodelist=nodes_top, node_color=node_colors, alpha=0.8)\n",
        "nx.draw_networkx_edges(G, pos, edge_color='gray', alpha=0.5)\n",
        "node_labels = nx.get_node_attributes(G, 'label')\n",
        "nx.draw_networkx_labels(G, pos, labels=node_labels, font_size=10, font_family='sans-serif')\n",
        "plt.axis('off')\n",
        "plt.figure(figsize=(30, 20))\n",
        "plt.show()\n"
      ],
      "metadata": {
        "id": "AeMMvwzfa8Ye"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import faiss\n",
        "import numpy as np\n",
        "from sentence_transformers import SentenceTransformer, util\n",
        "\n",
        "#On charge le modèle de transformation de phrases \"all-MiniLM-L6-v2\" à partir de la bibliothèque SentenceTransformer.\n",
        "\n",
        "model = SentenceTransformer('all-MiniLM-L6-v2')\n",
        "\n",
        "#le nombre de voisins les plus proches à récupérer.\n",
        "# Define the number of nearest neighbors to retrieve\n",
        "k = 5\n",
        "\n",
        "# Define the corpus of sentences\n",
        "corpus = matched_list[1:]\n",
        "\n",
        "# Encoder le corpus de phrases en utilisant le modèle de transformation de phrases.\n",
        "corpus_embeddings = model.encode(corpus)\n",
        "\n",
        "# Embed the query sentence\n",
        "query = matched_list[0]\n",
        "query_embedding = model.encode([query])\n",
        "\n",
        "# Create an index for the corpus embeddings\n",
        "#On crée un index FAISS pour les embeddings des phrases dans le corpus,\n",
        "#en utilisant l'algorithme \"IndexFlatIP\" pour calculer le produit scalaire entre les embeddings.\n",
        "d = len(corpus_embeddings[0])\n",
        "index = faiss.IndexFlatIP(d)\n",
        "index.add(np.array(corpus_embeddings))\n",
        "\n",
        "print(index)\n",
        "\n",
        "# Search for the k nearest neighbors\n",
        "D, I = index.search(np.array(query_embedding), k)\n",
        "\n",
        "# Print the results\n",
        "print(\"\\n\\n======================\\n\\n\")\n",
        "print(\"Query:\", query)\n",
        "print(\"\\nTop 5 most similar sentences in corpus:\")\n",
        "for i, idx in enumerate(I[0]):\n",
        "    print(corpus[idx], \"(Score: {:.4f})\".format(D[0][i]))"
      ],
      "metadata": {
        "id": "fqgV6nhLh2Lh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "list1 = [\"good job tracking sleep\",\n",
        "\"enter data back sleep\",\n",
        "\"total sleep time graph\",\n",
        "\"overall sleep patterns note\",\n",
        "\"able edit sleep times\",\n",
        "\"actual time records sleep\",\n",
        "\"simple sleep tracker fitness\", 'reminder automatic sleep detection',       'automatic activity tracker sleep',       'non food tracking barcode',       'select day week selector',       'regular meals have transferred',       'meals plus custom meals',       'motivational speech minimum interruption',       'persistent notification app running',       'paper plus follow work',       'daily activity rings resetting',       'achieve day from team',       'educational articles nutrition exercise',       'daily food track nutrition',       'food instance doesn track',       'daily track steps exercise',       'intake day track calories',       'breastfeeding moms for pregnancy',       'new achiever level badge',        'daily basis heart rate']"
      ],
      "metadata": {
        "id": "dLBFmHoJ8Rwd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "list2 =[\"good job tracking sleep\", \"total sleep time graph\", \"overall sleep patterns note\", \"own step feature edit\", \"skip login option watch\", \"actual step count ratio\", \"goals thanks perks far\", \"yearly challenge friends term\", \"yearly step km goal\", \"total steps miles kilometers\", \"yearly goals steps distance\", \"mandatory step verification glitches\", \"other sleep data sleep\", \"day sleep occurred night\", \"simple sleep tracker fitness\", \"figured sleep of turn\", \"total seconds month sleep\", \"morning and notice thinks\", \"exact route day app\", \"deep sleep stage hour\", \"choose sleep from notification\", \"real challenge ability export\", \"total duration speed tracker\", \"ran consecutive ever reaching\", \"decent step calorie counter\", \"more steps discrepancy app\"]"
      ],
      "metadata": {
        "id": "q332bR8m8cWR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn import metrics\n",
        "metrics.adjusted_rand_score(list1, list2)"
      ],
      "metadata": {
        "id": "q8filTe58y-_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "list1 = [\"good job tracking sleep\"]; list2 = [\"\"]"
      ],
      "metadata": {
        "id": "YXPB8NH19c9y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "list1 = [1, 1, 0, 0, 3, 3]; list2 = [1, 2, 0, 1, 2, 3]"
      ],
      "metadata": {
        "id": "o8Ixd5Hv99fM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "set1 = set(list1)\n",
        "set2 = set(list2)\n",
        "\n",
        "intersection = set1.intersection(set2)\n",
        "union = set1.union(set2)\n",
        "\n",
        "similarity = len(intersection) / len(union)\n",
        "\n",
        "print(f\"Jaccard similarity: {similarity}\")\n"
      ],
      "metadata": {
        "id": "QQX1gEsaDB2v"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}