05_embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "source": [
    "## Embeddings \n",
    "Esta seção mostra como usar embeddings para achar textos similares."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "gather": {
     "logged": 1674829364153
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from openai import AzureOpenAI\n",
    "from dotenv import load_dotenv\n",
    "import openai\n",
    "import re\n",
    "import requests\n",
    "import sys\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "load_dotenv()\n",
    "\n",
    "API_KEY = os.getenv(\"AZURE_OPENAI_API_KEY\",\"\").strip()\n",
    "assert API_KEY, \"ERROR: Azure OpenAI Key is missing\"\n",
    "\n",
    "RESOURCE_ENDPOINT = os.getenv(\"AZURE_OPENAI_API_BASE\",\"\").strip()\n",
    "assert RESOURCE_ENDPOINT, \"ERROR: Azure OpenAI Endpoint is missing\"\n",
    "assert \"openai.azure.com\" in RESOURCE_ENDPOINT.lower(), \"ERROR: Azure OpenAI Endpoint should be in the form: \\n\\n\\t<your unique endpoint identifier>.openai.azure.com\"\n",
    "\n",
    "client = AzureOpenAI(\n",
    "    api_version=\"2023-07-01-preview\",\n",
    "    azure_endpoint=RESOURCE_ENDPOINT,\n",
    "    api_key = API_KEY,\n",
    ")\n",
    "COMPLETIONS_MODEL = os.getenv('DEPLOYMENT_NAME')\n",
    "\n",
    "def cosine_similarity(a, b):\n",
    "    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "gather": {
     "logged": 1674829424097
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "text = 'the quick brown fox jumped over the lazy dog'\n",
    "\n",
    "model = 'text-embedding-ada-002'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "gather": {
     "logged": 1674829446467
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "client.embeddings.create(\n",
    "    input=text, model=model\n",
    ").data[0].embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "gather": {
     "logged": 1674829555255
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Similaridade entre 'automobile' e 'vehicle': 0.8662763107939466)\n",
      "Similaridade entre 'automobile' e 'carro': 0.8521030859141626)\n",
      "Similaridade entre 'automobile' e 'dinosaur': 0.7812571967097207)\n",
      "Similaridade entre 'automobile' e 'stick': 0.7237705785155591)\n",
      "Similaridade entre 'The cat says' e 'meow': 0.8590875195587717)\n",
      "Similaridade entre 'The cat says' e 'woof': 0.7833661082246537)\n",
      "Similaridade entre 'The cat says' e 'moo': 0.7663226212856344)\n",
      "Similaridade entre 'The dog says' e 'meow': 0.8174054195686926)\n",
      "Similaridade entre 'The dog says' e 'woof': 0.8297936141589077)\n",
      "Similaridade entre 'The dog says' e 'moo': 0.7902230641583795)\n",
      "Similaridade entre 'The cow says' e 'meow': 0.7803432402811546)\n",
      "Similaridade entre 'The cow says' e 'woof': 0.77463492589361)\n",
      "Similaridade entre 'The cow says' e 'moo': 0.856210470247614)\n",
      "Similaridade entre 'Messi' e 'gol': 0.7605945016523371)\n",
      "Similaridade entre 'Messi' e 'cesta': 0.7279251699633615)\n",
      "Similaridade entre 'Kobe Bryant' e 'goal': 0.7333416135784063)\n",
      "Similaridade entre 'Kobe Bryant' e 'basket': 0.7242608066857115)\n"
     ]
    }
   ],
   "source": [
    "# compare two words\n",
    "def compare(word1, word2):\n",
    "    word1_emb = client.embeddings.create(input=word1, model=model).data[0].embedding\n",
    "    word2_emb = client.embeddings.create(input=word2, model=model).data[0].embedding\n",
    "    similarity = cosine_similarity(word1_emb, word2_emb)\n",
    "    print(f\"Similaridade entre '{word1}' e '{word2}': {similarity})\")\n",
    "\n",
    "compare(\"automobile\", \"vehicle\")\n",
    "compare(\"automobile\", \"carro\")\n",
    "compare(\"automobile\", \"dinosaur\")\n",
    "compare(\"automobile\", \"stick\")\n",
    "\n",
    "compare(\"The cat says\", \"meow\")\n",
    "compare(\"The cat says\", \"woof\")\n",
    "compare(\"The cat says\", \"moo\")\n",
    "compare(\"The dog says\", \"meow\")\n",
    "compare(\"The dog says\", \"woof\")\n",
    "compare(\"The dog says\", \"moo\")\n",
    "compare(\"The cow says\", \"meow\")\n",
    "compare(\"The cow says\", \"woof\")\n",
    "compare(\"The cow says\", \"moo\")\n",
    "\n",
    "compare(\"Messi\", \"gol\")\n",
    "compare(\"Messi\", \"cesta\")\n",
    "\n",
    "compare(\"Kobe Bryant\", \"goal\")\n",
    "compare(\"Kobe Bryant\", \"dunk\")\n",
    "compare(\"Kobe Bryant\", \"three pointer\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Similaridade entre 'Kobe Bryant' e 'goal': 0.7333416135784063)\n",
      "Similaridade entre 'Kobe Bryant' e 'dunk': 0.7581180132101227)\n",
      "Similaridade entre 'Kobe Bryant' e 'three pointer': 0.7463325465727267)\n"
     ]
    }
   ],
   "source": [
    "compare(\"Kobe Bryant\", \"goal\")\n",
    "compare(\"Kobe Bryant\", \"dunk\")\n",
    "compare(\"Kobe Bryant\", \"three pointer\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "source": [
    "## Comparando artigos da CNN\n",
    "source: https://huggingface.co/datasets/cnn_dailymail\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "gather": {
     "logged": 1674831122093
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>articles</th>\n",
       "      <th>highligths</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BREMEN, Germany -- Carlos Alberto, who scored ...</td>\n",
       "      <td>Werder Bremen pay a club record $10.7 million ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(CNN) -- Football superstar, celebrity, fashio...</td>\n",
       "      <td>Beckham has agreed to a five-year contract wit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LOS ANGELES, California (CNN) -- Youssif, the ...</td>\n",
       "      <td>Boy on meeting Spider-Man: \"It was my favorite...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            articles  \\\n",
       "0  BREMEN, Germany -- Carlos Alberto, who scored ...   \n",
       "1  (CNN) -- Football superstar, celebrity, fashio...   \n",
       "2  LOS ANGELES, California (CNN) -- Youssif, the ...   \n",
       "\n",
       "                                          highligths  \n",
       "0  Werder Bremen pay a club record $10.7 million ...  \n",
       "1  Beckham has agreed to a five-year contract wit...  \n",
       "2  Boy on meeting Spider-Man: \"It was my favorite...  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "cnn_daily_articles = ['BREMEN, Germany -- Carlos Alberto, who scored in FC Porto\\'s Champions League final victory against Monaco in 2004, has joined Bundesliga club Werder Bremen for a club record fee of 7.8 million euros ($10.7 million). Carlos Alberto enjoyed success at FC Porto under Jose Mourinho. \"I\\'m here to win titles with Werder,\" the 22-year-old said after his first training session with his new club. \"I like Bremen and would only have wanted to come here.\" Carlos Alberto started his career with Fluminense, and helped them to lift the Campeonato Carioca in 2002. In January 2004 he moved on to FC Porto, who were coached by José Mourinho, and the club won the Portuguese title as well as the Champions League. Early in 2005, he moved to Corinthians, where he impressed as they won the Brasileirão,but in 2006 Corinthians had a poor season and Carlos Alberto found himself at odds with manager, Emerson Leão. Their poor relationship came to a climax at a Copa Sul-Americana game against Club Atlético Lanús, and Carlos Alberto declared that he would not play for Corinthians again while Leão remained as manager. Since January this year he has been on loan with his first club Fluminense. Bundesliga champions VfB Stuttgart said on Sunday that they would sign a loan agreement with Real Zaragoza on Monday for Ewerthon, the third top Brazilian player to join the German league in three days. A VfB spokesman said Ewerthon, who played in the Bundesliga for Borussia Dortmund from 2001 to 2005, was expected to join the club for their pre-season training in Austria on Monday. On Friday, Ailton returned to Germany where he was the league\\'s top scorer in 2004, signing a one-year deal with Duisburg on a transfer from Red Star Belgrade. E-mail to a friend .',\n",
    "                        '(CNN) -- Football superstar, celebrity, fashion icon, multimillion-dollar heartthrob. Now, David Beckham is headed for the Hollywood Hills as he takes his game to U.S. Major League Soccer. CNN looks at how Bekham fulfilled his dream of playing for Manchester United, and his time playing for England. The world\\'s famous footballer has begun a five-year contract with the Los Angeles Galaxy team, and on Friday Beckham will meet the press and reveal his new shirt number. This week, we take an in depth look at the life and times of Beckham, as CNN\\'s very own \"Becks,\" Becky Anderson, sets out to examine what makes the man tick -- as footballer, fashion icon and global phenomenon. It\\'s a long way from the streets of east London to the Hollywood Hills and Becky charts Beckham\\'s incredible rise to football stardom, a journey that has seen his skills grace the greatest stages in world soccer. She goes in pursuit of the current hottest property on the sports/celebrity circuit in the U.S. and along the way explores exactly what\\'s behind the man with the golden boot. CNN will look back at the life of Beckham, the wonderfully talented youngster who fulfilled his dream of playing for Manchester United, his marriage to pop star Victoria, and the trials and tribulations of playing for England. We\\'ll look at the highs (scoring against Greece), the lows (being sent off during the World Cup), the Man. U departure for the Galacticos of Madrid -- and now the Home Depot stadium in L.A. We\\'ll ask how Beckham and his family will adapt to life in Los Angeles -- the people, the places to see and be seen and the celebrity endorsement. Beckham is no stranger to exposure. He has teamed with Reggie Bush in an Adidas commercial, is the face of Motorola, is the face on a PlayStation game and doesn\\'t need fashion tips as he has his own international clothing line. But what does the star couple need to do to become an accepted part of Tinseltown\\'s glitterati? The road to major league football in the U.S.A. is a well-worn route for some of the world\\'s greatest players. We talk to some of the former greats who came before him and examine what impact these overseas stars had on U.S. soccer and look at what is different now. We also get a rare glimpse inside the David Beckham academy in L.A, find out what drives the kids and who are their heroes. The perception that in the U.S.A. soccer is a \"game for girls\" after the teenage years is changing. More and more young kids are choosing the European game over the traditional U.S. sports. E-mail to a friend .',\n",
    "                        'LOS ANGELES, California (CNN) -- Youssif, the 5-year-old burned Iraqi boy, rounded the corner at Universal Studios when suddenly the little boy hero met his favorite superhero. Youssif has always been a huge Spider-Man fan. Meeting him was \"my favorite thing,\" he said. Spider-Man was right smack dab in front of him, riding a four-wheeler amid a convoy of other superheroes. The legendary climber of buildings and fighter of evil dismounted, walked over to Youssif and introduced himself. Spidey then gave the boy from a far-away land a gentle hug, embracing him in his iconic blue and red tights. He showed Youssif a few tricks, like how to shoot a web from his wrist. Only this time, no web was spun. \"All right Youssif!\" Spider-Man said after the boy mimicked his wrist movement. Other superheroes crowded around to get a closer look. Even the Green Goblin stopped his villainous ways to tell the boy hi. Youssif remained unfazed. He didn\\'t take a liking to Spider-Man\\'s nemesis. Spidey was just too cool. \"It was my favorite thing,\" the boy said later. \"I want to see him again.\" He then felt compelled to add: \"I know it\\'s not the real Spider-Man.\" This was the day of dreams when the boy\\'s nightmares were, at least temporarily, forgotten. He met SpongeBob, Lassie and a 3-year-old orangutan named Archie. The hairy, brownish-red primate took to the boy, grabbing his hand and holding it. Even when Youssif pulled away, Archie would inch his hand back toward the boy\\'s and then snatch it. See Youssif enjoy being a boy again » . The boy giggled inside a play area where sponge-like balls shot out of toy guns. It was a far different artillery than what he was used to seeing in central Baghdad, as recently as a week ago. He squealed with delight and raced around the room collecting as many balls as he could. He rode a tram through the back stages at Universal Studios. At one point, the car shook. Fire and smoke filled the air, debris cascaded down and a big rig skidded toward the vehicle. The boy and his family survived the pretend earthquake unscathed. \"Even I was scared,\" the dad said. \"Well, I wasn\\'t,\" Youssif replied. The father and mother grinned from ear to ear throughout the day. Youssif pushed his 14-month-old sister, Ayaa, in a stroller. \"Did you even need to ask us if we were interested in coming here?\" Youssif\\'s father said in amazement. \"Other than my wedding day, this is the happiest day of my life,\" he said. Just a day earlier, the mother and father talked about their journey out of Iraq and to the United States. They also discussed that day nine months ago when masked men grabbed their son outside the family home, doused him in gas and set him on fire. His mother heard her boy screaming from inside. The father sought help for his boy across Baghdad, but no one listened. He remembers his son\\'s two months of hospitalization. The doctors didn\\'t use anesthetics. He could hear his boy\\'s piercing screams from the other side of the hospital. Watch Youssif meet his doctor and play with his little sister » . The father knew that speaking to CNN would put his family\\'s lives in jeopardy. The possibility of being killed was better than seeing his son suffer, he said. \"Anything for Youssif,\" he said. \"We had to do it.\" They described a life of utter chaos in Baghdad. Neighbors had recently given birth to a baby girl. Shortly afterward, the father was kidnapped and killed. Then, there was the time when some girls wore tanktops and jeans. They were snatched off the street by gunmen. The stories can be even more gruesome. The couple said they had heard reports that a young girl was kidnapped and beheaded --and her killers sewed a dog\\'s head on the corpse and delivered it to her family\\'s doorstep. \"These are just some of the stories,\" said Youssif\\'s mother, Zainab. Under Saddam Hussein, there was more security and stability, they said. There was running water and electricity most of the time. But still life was tough under the dictator, like the time when Zainab\\'s uncle disappeared and was never heard from again after he read a \"religious book,\" she said. Sitting in the parking lot of a Target in suburban Los Angeles, Youssif\\'s father watched as husbands and wives, boyfriends and girlfriends, parents and their children, came and went. Some held hands. Others smiled and laughed. \"Iraq finished,\" he said in what few English words he knows. He elaborated in Arabic: His homeland won\\'t be enjoying such freedoms anytime soon. It\\'s just not possible. Too much violence. Too many killings. His two children have only seen war. But this week, the family has seen a much different side of America -- an outpouring of generosity and a peaceful nation at home. \"It\\'s been a dream,\" the father said. He used to do a lot of volunteer work back in Baghdad. \"Maybe that\\'s why I\\'m being helped now,\" the father said. At Universal Studios, he looked out across the valley below. The sun glistened off treetops and buildings. It was a picturesque sight fit for a Hollywood movie. \"Good America, good America,\" he said in English. E-mail to a friend . CNN\\'s Arwa Damon contributed to this report.'\n",
    "]\n",
    "\n",
    "cnn_daily_article_highlights = ['Werder Bremen pay a club record $10.7 million for Carlos Alberto .\\nThe Brazilian midfielder won the Champions League with FC Porto in 2004 .\\nSince January he has been on loan with his first club, Fluminense .',\n",
    "                                'Beckham has agreed to a five-year contract with Los Angeles Galaxy .\\nNew contract took effect July 1, 2007 .\\nFormer English captain to meet press, unveil new shirt number Friday .\\nCNN to look at Beckham as footballer, fashion icon and global phenomenon .',\n",
    "                                'Boy on meeting Spider-Man: \"It was my favorite thing\"\\nYoussif also met SpongeBob, Lassie and an orangutan at Universal Studios .\\nDad: \"Other than my wedding day, this is the happiest day of my life\"' \n",
    "]\n",
    "\n",
    "cnn_df = pd.DataFrame({\"articles\":cnn_daily_articles, \"highligths\":cnn_daily_article_highlights})\n",
    "\n",
    "cnn_df.head()                      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "gather": {
     "logged": 1674831294043
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7577417762458007\n",
      "0.6935440260324772\n",
      "0.7586446040644623\n"
     ]
    }
   ],
   "source": [
    "article1_embedding    = client.embeddings.create(input=cnn_df.articles.iloc[0], model=model).data[0].embedding\n",
    "article2_embedding    = client.embeddings.create(input=cnn_df.articles.iloc[1], model=model).data[0].embedding\n",
    "article3_embedding    = client.embeddings.create(input=cnn_df.articles.iloc[2], model=model).data[0].embedding\n",
    "\n",
    "print(cosine_similarity(article1_embedding, article2_embedding))\n",
    "print(cosine_similarity(article1_embedding, article3_embedding))\n",
    "print(cosine_similarity(article2_embedding, article3_embedding))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernel_info": {
   "name": "python310-sdkv2"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "microsoft": {
   "host": {
    "AzureML": {
     "notebookHasBeenCompleted": true
    }
   }
  },
  "nteract": {
   "version": "nteract-front-end@1.0.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}