Extracting_textual_representations_Bert_Exercise.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Extracting_textual_representations_Bert_Exercise.ipynb",
      "private_outputs": true,
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "msi7t_qjcTqx"
      },
      "outputs": [],
      "source": [
        "!pip -q install transformers"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Working with Bert Embeddings"
      ],
      "metadata": {
        "id": "dVkE060qdghQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import numpy as np\n",
        "from transformers import AutoTokenizer, AutoModel\n",
        "from tqdm import tqdm\n",
        "\n"
      ],
      "metadata": {
        "id": "r6aWRO6kduJ6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Loading Bert embeddings"
      ],
      "metadata": {
        "id": "8RJ2HslZg9kC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_ckpt = \"distilbert-base-uncased\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_ckpt)\n",
        "model = AutoModel.from_pretrained(model_ckpt, output_hidden_states=True).to('cpu')\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_ckpt)\n",
        "model.eval()"
      ],
      "metadata": {
        "id": "LaBKnQswf-zQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text = \"After stealing money from the bank vault, the bank robber was seen fishing on the Nile river bank.\"\n",
        "# try to tokenize the text, and extract the input IDs\n",
        "inputs = tokenizer(text, return_tensors=\"pt\", add_special_tokens=True)\n",
        "\n",
        "# try to get the token ids from the tokeniser\n",
        "\n",
        "#your output: [101,2044,11065,2769,...,1012,102]\n",
        "tokens_ids =  inputs[0].ids \n",
        "\n",
        "segments_ids = [1] * len(tokens_ids)\n",
        "\n",
        "all_tokens = inputs[0].tokens\n",
        "\n",
        "## Converting to Pytorch Tensors\n",
        "tokens_tensor = torch.tensor([tokens_ids])\n",
        "segments_tensors = torch.tensor([segments_ids])\n",
        "\n",
        "print('tokens tensor shape: ',tokens_tensor.shape)\n",
        "print('Segments tensor shape: ',segments_tensors.shape)\n",
        "\n",
        "## Output:\n",
        "#tokens tensor shape:  torch.Size([1, 22])\n",
        "#Segments tensor shape:  torch.Size([1, 22])"
      ],
      "metadata": {
        "id": "a7fcekbEhJTd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Extract the Embeddings"
      ],
      "metadata": {
        "id": "-3LSzsU7olNg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# how can you get all the hidden states from the model ?\n",
        "with torch.no_grad():\n",
        "  outputs = model(tokens_tensor, segments_tensors)\n",
        "  hidden_states = outputs[1]\n",
        "hidden_states[0].shape"
      ],
      "metadata": {
        "id": "kiH2zasWowVL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "## try to inspect the hidden_state  shape\n",
        "## output: torch.Size([1, 22, 768])"
      ],
      "metadata": {
        "id": "SALMogUlxj3t"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print (\"Number of layers:\", len(hidden_states), \"  (initial embeddings + 6 distilbert  layers)\")\n",
        "layer_i = 0\n",
        "\n",
        "print (\"Number of batches:\", len(hidden_states[layer_i]))\n",
        "batch_i = 0\n",
        "\n",
        "print (\"Number of tokens:\", len(hidden_states[layer_i][batch_i]))\n",
        "token_i = 0\n",
        "\n",
        "print (\"Number of hidden units:\", len(hidden_states[layer_i][batch_i][token_i]))"
      ],
      "metadata": {
        "id": "9DWgzb3vxlri"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#For the 4th token in our sentence, select its feature values from layer 4.\n",
        "token_i = 4\n",
        "layer_i = 4\n",
        "vec = hidden_states[layer_i][batch_i][token_i]"
      ],
      "metadata": {
        "id": "hOvojBHdzTpW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vec.size()"
      ],
      "metadata": {
        "id": "LKXIku2Yz82U"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Concatenate the tensors for all layers.\n",
        "# create a new dimension in the tensor.\n",
        "token_embeddings = torch.stack(hidden_states, dim=0)\n",
        "\n",
        "token_embeddings.size()"
      ],
      "metadata": {
        "id": "2wxH_kFqz_Az"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Remove dimension 1, the \"batches\".\n",
        "token_embeddings = torch.squeeze(token_embeddings,dim=1)\n",
        "token_embeddings.size()"
      ],
      "metadata": {
        "id": "ydNTqsmc2Tgh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# you nees to Swap dimensions 0 and 1.\n",
        "token_embeddings = token_embeddings.permute(1,0,2)\n",
        "token_embeddings.size()\n",
        "#output: torch.Size([22, 7, 768])"
      ],
      "metadata": {
        "id": "Yi6DwNR72d0Z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Extracting Word Embeddings \n",
        "### 1 - Word Representation (concat) \n"
      ],
      "metadata": {
        "id": "FdXNAfBS2sNQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "## write a function that concatenate the representation of tokens from the 4 last layers\n",
        "token_vecs_cat = []\n",
        "for token in token_embeddings:\n",
        "    cat_vec = torch.cat((token[-1],token[-2],token[-3],token[-4]),dim=0)\n",
        "    token_vecs_cat.append(cat_vec)\n",
        "print ('Shape is: %d x %d' % (len(token_vecs_cat), len(token_vecs_cat[0])))\n",
        "\n",
        "#output: Shape is: 22 x 3072"
      ],
      "metadata": {
        "id": "NLf7yCzi22Hm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "###2 -Word Representation (sum)"
      ],
      "metadata": {
        "id": "dadwV5py3iwY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "## write a function that sum the representation of tokens from the 4 last layers\n",
        "token_vecs_sum = []\n",
        "for token in token_embeddings:\n",
        "    sum_vec = torch.sum(token[-4:],dim=0)\n",
        "    token_vecs_sum.append(sum_vec)\n",
        "print ('Shape is: %d x %d' % (len(token_vecs_sum), len(token_vecs_sum[0])))\n",
        "\n",
        "#output: Shape is: 22 x 768"
      ],
      "metadata": {
        "id": "Ty7a4krf3vD1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "###3 - Sentence Representation(mean)"
      ],
      "metadata": {
        "id": "YKCyBvVUMnNd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# `hidden_states` has shape [7 x 1 x 22 x 768]\n",
        "\n",
        "# `token_vecs` is a tensor with shape [22 x 768]\n",
        "token_vecs = hidden_states[0][0]\n",
        "\n",
        "# Calculate the average of all 22 token vectors.\n",
        "sentence_embedding = torch.mean(token_vecs,dim=0)"
      ],
      "metadata": {
        "id": "IA4Z-tNvMzpL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print (\"Sentence Embedding Shape:\", sentence_embedding.size())"
      ],
      "metadata": {
        "id": "RpeyZg34NUTh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "ZFWiIpHeNpqO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Evaluating Context dependent Representations"
      ],
      "metadata": {
        "id": "UBaP9R1bSB2P"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for i, token_str in enumerate(all_tokens):\n",
        "  print (i, '==>' , token_str)"
      ],
      "metadata": {
        "id": "rNaI4hCWNy7E"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from scipy.spatial.distance import cosine\n",
        "\n",
        "def get_similarity(vec_1,vec_2):\n",
        "  return 1 - cosine(vec_1, vec_2)\n"
      ],
      "metadata": {
        "id": "91-OVUa_Vitx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "## use the above function to calculate semantic similarity between [10 ==> bank] and [6 ==> bank] as well as \n",
        "#between [10 ==> bank] and [19 ==> bank]\n",
        "## does the model cupture this similarity\n",
        "\n",
        "get_similarity(token_vecs_sum[10],token_vecs_sum[6])"
      ],
      "metadata": {
        "id": "66BgKhAmXUpQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "get_similarity(token_vecs_sum[10],token_vecs_sum[19])"
      ],
      "metadata": {
        "id": "zmgzOC39YyPI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "get_similarity(token_vecs_cat[10],token_vecs_cat[6])"
      ],
      "metadata": {
        "id": "mdmwJGMBEkQi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "get_similarity(token_vecs_cat[10],token_vecs_cat[19])"
      ],
      "metadata": {
        "id": "x33owshkEtgi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q sentence-transformers"
      ],
      "metadata": {
        "id": "ARTWk_vsE2OB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sentence_transformers import SentenceTransformer\n",
        "model = SentenceTransformer('paraphrase-MiniLM-L6-v2')\n"
      ],
      "metadata": {
        "id": "6JNUWB-oFQ-x"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Our sentences we like to encode\n",
        "sentences = ['This framework generates embeddings for each input sentence',\n",
        "    'Sentences are passed as a list of string.',\n",
        "    'The quick brown fox jumps over the lazy dog.']"
      ],
      "metadata": {
        "id": "MUa1PDmLFeTh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Sentences are encoded by calling model.encode()\n",
        "embeddings = model.encode(sentences)\n",
        "embeddings.shape"
      ],
      "metadata": {
        "id": "542HPFjBFtnh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# #Print the embeddings\n",
        "# for sentence, embedding in zip(sentences, embeddings):\n",
        "#     print(\"Sentence:\", sentence)\n",
        "#     print(\"Embedding:\", embedding)\n",
        "#     print(\"\")"
      ],
      "metadata": {
        "id": "9Kd4wAdDFzVC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.cluster import AgglomerativeClustering"
      ],
      "metadata": {
        "id": "vVilCHWOGA6m"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np"
      ],
      "metadata": {
        "id": "7BKL0mShGpFS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "embedder = SentenceTransformer('sentence-transformers/stsb-bert-base')"
      ],
      "metadata": {
        "id": "qX6JZm3pGxMB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "corpus = ['A man is eating food.',\n",
        "          'A man is eating a piece of bread.',\n",
        "          'A man is eating pasta.',\n",
        "          'The girl is carrying a baby.',\n",
        "          'The baby is carried by the woman',\n",
        "          'A man is riding a horse.',\n",
        "          'A man is riding a white horse on an enclosed ground.',\n",
        "          'A monkey is playing drums.',\n",
        "          'Someone in a gorilla costume is playing a set of drums.',\n",
        "          'A cheetah is running behind its prey.',\n",
        "          'A cheetah chases prey on across a field.'\n",
        "          ]"
      ],
      "metadata": {
        "id": "1RzJkftnHH8R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "corpus_embeddings = embedder.encode(corpus)\n",
        "corpus_embeddings.shape"
      ],
      "metadata": {
        "id": "JUnDbvG3HMAB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Normalize the embeddings to unit length\n",
        "corpus_embeddings = corpus_embeddings /  np.linalg.norm(corpus_embeddings, axis=1, keepdims=True)"
      ],
      "metadata": {
        "id": "nEUbyqetHUDB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Perform kmean clustering\n",
        "clustering_model = AgglomerativeClustering(n_clusters=None, distance_threshold=1.5) #, affinity='cosine', linkage='average', distance_threshold=0.4)\n",
        "clustering_model.fit(corpus_embeddings)\n",
        "cluster_assignment = clustering_model.labels_"
      ],
      "metadata": {
        "id": "u0lMT_hcHbGB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cluster_assignment"
      ],
      "metadata": {
        "id": "hRah3pwRHerC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "clustered_sentences = {}\n",
        "for sentence_id, cluster_id in enumerate(cluster_assignment):\n",
        "    if cluster_id not in clustered_sentences:\n",
        "        clustered_sentences[cluster_id] = []\n",
        "\n",
        "    clustered_sentences[cluster_id].append(corpus[sentence_id])\n",
        "\n",
        "for i, cluster in clustered_sentences.items():\n",
        "    print(\"Cluster \", i+1)\n",
        "    print(cluster)\n",
        "    print(\"\")"
      ],
      "metadata": {
        "id": "RuXaXj5ZHp3y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "4gva3DfC-oVl"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}