HomeMatch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a108af04",
   "metadata": {},
   "source": [
    "This is a starter notebook for the project, you'll have to import the libraries you'll need, you can find a list of the ones available in this workspace in the requirements.txt file in this workspace. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad9ddb5c",
   "metadata": {},
   "source": [
    "# Import librairies\n",
    "- please note chromadb was pinned to 0.4.15 to be compatible with langchain OpenAIEmbeddings parameters (incompatibility issue https://github.com/langchain-ai/langchain/discussions/14411)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 540,
   "id": "6554fd4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, json\n",
    "from IPython.display import Markdown, display\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pprint import pprint\n",
    "\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "# load up the .env file with secrets\n",
    "_ = load_dotenv(find_dotenv())\n",
    "\n",
    "from pydantic import BaseModel, Field, NonNegativeInt\n",
    "from typing import List, Optional, Type\n",
    "\n",
    "from langchain.llms import OpenAI\n",
    "from langchain.chat_models import ChatOpenAI\n",
    "from langchain.vectorstores import Chroma\n",
    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
    "from langchain.output_parsers import PydanticOutputParser\n",
    "from langchain.document_loaders import CSVLoader  #JSONLoader\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter, CharacterTextSplitter\n",
    "from langchain.output_parsers import ResponseSchema\n",
    "from langchain.output_parsers import StructuredOutputParser\n",
    "from langchain.prompts import (ChatPromptTemplate,\n",
    "                               PromptTemplate,\n",
    "                               SystemMessagePromptTemplate,\n",
    "                               AIMessagePromptTemplate,\n",
    "                               HumanMessagePromptTemplate,\n",
    "                               )\n",
    "from langchain.schema import AIMessage, HumanMessage, SystemMessage, FunctionMessage\n",
    "from langchain.utils.openai_functions import convert_pydantic_to_openai_function\n",
    "from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser\n",
    "from langchain.tools import BaseTool, format_tool_to_openai_function\n",
    "from langchain import LLMChain\n",
    "\n",
    "import openai"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e924758a",
   "metadata": {},
   "source": [
    "# Key modules and parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a0574b87",
   "metadata": {},
   "outputs": [],
   "source": [
    "# colors definition\n",
    "ENDC = '\\33[0m'\n",
    "RED = '\\33[31m'\n",
    "GREEN = '\\33[32m'\n",
    "YELLOW = '\\33[33m'\n",
    "VIOLET = '\\33[35m'\n",
    "CYAN = '\\33[36m'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "e6d60a5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instantiate an OpenAI API client using langchain\n",
    "instruct_llm = OpenAI(model=\"gpt-3.5-turbo-instruct\", \n",
    "             openai_api_key=os.getenv('OPENAI_API_KEY'),\n",
    "             temperature=0, \n",
    "             max_tokens=3500)\n",
    "\n",
    "chat_llm = ChatOpenAI(model=\"gpt-3.5-turbo\", \n",
    "             openai_api_key=os.getenv('OPENAI_API_KEY'),\n",
    "             temperature=0, \n",
    "             max_tokens=3500)\n",
    "\n",
    "import chromadb\n",
    "persist_directory = './data/vectordb'\n",
    "client = chromadb.PersistentClient(path=persist_directory)\n",
    "\n",
    "embeddings_model = OpenAIEmbeddings(openai_api_key=os.getenv('OPENAI_API_KEY'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bcd63aa7",
   "metadata": {},
   "source": [
    "# STEP 1 - GENERATE SYNTHETIC REAL ESTATE LISTINGS\n",
    "- we want to generate synthetic listings to use as our test database\n",
    "- we will use chatGPT to generate these listings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ad94ae57",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The output should be formatted as a JSON instance that conforms to the JSON schema below.\n",
      "\n",
      "As an example, for the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, \"required\": [\"foo\"]}\n",
      "the object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} is not well-formatted.\n",
      "\n",
      "Here is the output schema:\n",
      "```\n",
      "{\"$defs\": {\"listings_description_model\": {\"properties\": {\"listings_headline\": {\"description\": \"property headline\", \"title\": \"Listings Headline\", \"type\": \"string\"}, \"listings_neighborhood\": {\"description\": \"property neighborhood where the home is located\", \"title\": \"Listings Neighborhood\", \"type\": \"string\"}, \"listings_price\": {\"description\": \"property price\", \"minimum\": 0, \"title\": \"Listings Price\", \"type\": \"integer\"}, \"listings_bedrooms\": {\"description\": \"property number of bedrooms\", \"minimum\": 0, \"title\": \"Listings Bedrooms\", \"type\": \"integer\"}, \"listings_bathrooms\": {\"description\": \"property number of bathrooms\", \"minimum\": 0, \"title\": \"Listings Bathrooms\", \"type\": \"integer\"}, \"listings_house_size\": {\"description\": \"property house size\", \"minimum\": 0, \"title\": \"Listings House Size\", \"type\": \"integer\"}, \"listings_description\": {\"description\": \"property description following the guidelines\", \"title\": \"Listings Description\", \"type\": \"string\"}, \"listings_neighborhood_description\": {\"description\": \"property neighborhood description\", \"title\": \"Listings Neighborhood Description\", \"type\": \"string\"}}, \"required\": [\"listings_headline\", \"listings_neighborhood\", \"listings_price\", \"listings_bedrooms\", \"listings_bathrooms\", \"listings_house_size\", \"listings_description\", \"listings_neighborhood_description\"], \"title\": \"listings_description_model\", \"type\": \"object\"}}, \"properties\": {\"listings_headline\": {\"description\": \"property headline\", \"title\": \"Listings Headline\", \"type\": \"string\"}, \"listings_neighborhood\": {\"description\": \"property neighborhood where the home is located\", \"title\": \"Listings Neighborhood\", \"type\": \"string\"}, \"listings_price\": {\"description\": \"property price\", \"minimum\": 0, \"title\": \"Listings Price\", \"type\": \"integer\"}, \"listings_bedrooms\": {\"description\": \"property number of bedrooms\", \"minimum\": 0, \"title\": \"Listings Bedrooms\", \"type\": \"integer\"}, \"listings_bathrooms\": {\"description\": \"property number of bathrooms\", \"minimum\": 0, \"title\": \"Listings Bathrooms\", \"type\": \"integer\"}, \"listings_house_size\": {\"description\": \"property house size\", \"minimum\": 0, \"title\": \"Listings House Size\", \"type\": \"integer\"}, \"listings_description\": {\"description\": \"property description following the guidelines\", \"title\": \"Listings Description\", \"type\": \"string\"}, \"listings_neighborhood_description\": {\"description\": \"property neighborhood description\", \"title\": \"Listings Neighborhood Description\", \"type\": \"string\"}, \"list_of_listings\": {\"description\": \"list of property descriptions following the guidelines\", \"items\": {\"$ref\": \"#/$defs/listings_description_model\"}, \"title\": \"List Of Listings\", \"type\": \"array\"}}, \"required\": [\"listings_headline\", \"listings_neighborhood\", \"listings_price\", \"listings_bedrooms\", \"listings_bathrooms\", \"listings_house_size\", \"listings_description\", \"listings_neighborhood_description\", \"list_of_listings\"]}\n",
      "```\n"
     ]
    }
   ],
   "source": [
    "# Prepare property description examples for two-shot learning\n",
    "listing_examples = ['''Headline: A 3-bedroom, 2-bathroom home in Green Oaks\n",
    "Neighborhood: Green Oaks\n",
    "Price: €1,200,000\n",
    "Bedrooms: 3\n",
    "Bathrooms: 2\n",
    "House Size (sqm): 110\n",
    "\n",
    "Description: Welcome to this eco-friendly oasis nestled in the heart of Green Oaks. This charming 3-bedroom, 2-bathroom home boasts energy-efficient features such as solar panels and a well-insulated structure. Natural light floods the living spaces, highlighting the beautiful hardwood floors and eco-conscious finishes. The open-concept kitchen and dining area lead to a spacious backyard with a vegetable garden, perfect for the eco-conscious family. Embrace sustainable living without compromising on style in this Green Oaks gem.\n",
    "\n",
    "Neighborhood Description: Green Oaks is a close-knit, environmentally-conscious community with access to organic grocery stores, community gardens, and bike paths. Take a stroll through the nearby Green Oaks Park or grab a cup of coffee at the cozy Green Bean Cafe. With easy access to public transportation and bike lanes, commuting is a breeze.''',\n",
    "'''Headline: Charming Oasis in Neuilly\n",
    "Neighborhood: Neuilly\n",
    "Price: €2,000,000\n",
    "Bedrooms: 5\n",
    "Bathrooms: 2\n",
    "House Size (sqm): 180\n",
    "\n",
    "Description: Discover this enchanting family home nestled in the heart of Neuilly. With its timeless elegance and modern amenities, it’s the perfect sanctuary for your family.\n",
    "\n",
    "Neighborhood Description: Neuilly offers a serene lifestyle, top-rated schools, and picturesque parks. Enjoy the tranquility while being conveniently close to amenities.\n",
    "''']\n",
    "# convert examples to one string\n",
    "listing_samples = '\\n---------------------------------\\n'.join(listing_examples)\n",
    "\n",
    "# Pydantic models for the listings data\n",
    "class listings_description_model(BaseModel):\n",
    "    listings_headline: str = Field(description='property headline')\n",
    "    listings_neighborhood: str = Field(description='property neighborhood where the home is located')\n",
    "    listings_price: NonNegativeInt = Field(description='property price')\n",
    "    listings_bedrooms: NonNegativeInt = Field(description='property number of bedrooms')\n",
    "    listings_bathrooms: NonNegativeInt = Field(description='property number of bathrooms')\n",
    "    listings_house_size: NonNegativeInt = Field(description='property house size')\n",
    "    listings_description: str = Field(description='property description following the guidelines')\n",
    "    listings_neighborhood_description: str = Field(description='property neighborhood description')\n",
    "\n",
    "class listings_descriptions(listings_description_model):\n",
    "    list_of_listings: List[listings_description_model] = Field(description='list of property descriptions following the guidelines')\n",
    "\n",
    "parser = PydanticOutputParser(pydantic_object=listings_descriptions)\n",
    "print(parser.get_format_instructions())\n",
    "\n",
    "\n",
    "# Prompt to generate synthetic listing data using examples and pydantic model\n",
    "template_genlistings = '''You are a real estate agent tasked with creating property listings for a variety of homes in different neighborhoods. Your goal is to craft compelling descriptions that entice potential buyers. Follow these guidelines for each listing:\n",
    "1. Headline: Begin with an attention-grabbing opening sentence that captures the viewer's interest in the home.\n",
    "2. In Bullet points:\n",
    "    - Neighborhood: The name of the neighborhood where the home is located.\n",
    "    - **Price**: Mention the price.\n",
    "    - **Bedrooms**: Specify the number of bedrooms (e.g., 5).\n",
    "    - **Bathrooms**: Specify the number of bathrooms (e.g., 2).\n",
    "    - **House Size in sqm**: Provide the exact square footage (e.g., 150).\n",
    "3. Property Description: Include a brief description of the home and its features.\n",
    "4. Neighborhood description: Briefly describe the neighborhood.\n",
    "\n",
    "========== EXEMPLES ==========\n",
    "{listing_samples}\n",
    "========== END OF EXAMPLES ==========\n",
    "\n",
    "Remember to be succinct, informative, and persuasive.\n",
    "**Only use the requested format.**\n",
    "{format_instructions}\n",
    "'''\n",
    "\n",
    "prompt = PromptTemplate.from_template(template_genlistings,\n",
    "                                      partial_variables={'format_instructions':parser.get_format_instructions})\n",
    "\n",
    "system_prompt = prompt.format(listing_samples=listing_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "5de05684",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Call GPT-3 to generate 15 listings following above guidelines.\n",
    "response = openai.ChatCompletion.create(\n",
    "          model=\"gpt-3.5-turbo\",\n",
    "          messages=[\n",
    "          {\n",
    "            \"role\": \"system\",\n",
    "            \"content\": system_prompt\n",
    "          },\n",
    "          {\n",
    "            \"role\": \"user\",\n",
    "            \"content\": \"Generate 15 listings following these guidelines.\"\n",
    "          }\n",
    "          ],\n",
    "        temperature=0,\n",
    "        max_tokens=4000,\n",
    "        )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "410283fd",
   "metadata": {},
   "source": [
    "#### RUBRIC #1 Generating Real Estate Listings with an LLM"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99e6938c",
   "metadata": {},
   "source": [
    "- post processing to extract the listings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "878808f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"listings_headline\": \"Modern Luxury in Central Park\",\n",
      "    \"listings_neighborhood\": \"Central Park\",\n",
      "    \"listings_price\": 3500000,\n",
      "    \"listings_bedrooms\": 4,\n",
      "    \"listings_bathrooms\": 3,\n",
      "    \"listings_house_size\": 280,\n",
      "    \"listings_description\": \"Indulge in the epitome of modern luxury with this stunning 4-bedroom, 3-bathroom home in Central Park. Boasting high-end finishes, a gourmet kitchen, and panoramic views of the city skyline, this property offers a sophisticated urban lifestyle.\",\n",
      "    \"listings_neighborhood_description\": \"Central Park is a vibrant neighborhood known for its upscale dining, shopping, and cultural attractions. Enjoy easy access to the park for leisurely strolls and outdoor activities.\"\n",
      "},\n",
      "{\n",
      "    \"listings_headline\": \"Seaside Retreat in Ocean View\",\n",
      "    \"listings_neighborhood\": \"Ocean View\",\n",
      "    \"listings_price\": 1800000,\n",
      "    \"listings_bedrooms\": 3,\n",
      "    \"listings_bathrooms\": 2,\n",
      "    \"listings_house_size\": 200,\n",
      "    \"listings_description\": \"Escape to this charming seaside retreat in Ocean View. With 3 bedrooms, 2 bathrooms, and a spacious deck overlooking the ocean, this home offers a perfect blend of coastal living and modern comfort.\",\n",
      "    \"listings_neighborhood_description\": \"Ocean View is a picturesque coastal community with sandy beaches, seafood restaurants, and a laid-back atmosphere. Experience the tranquility of seaside living in this charming neighborhood.\"\n",
      "},\n",
      "{\n",
      "    \"listings_headline\": \"Historic Elegance in Heritage District\",\n",
      "    \"listings_neighborhood\": \"Heritage District\",\n",
      "    \"listings_price\": 1200000,\n",
      "    \"listings_bedrooms\": 5,\n",
      "    \"listings_bathrooms\": 4,\n",
      "    \"listings_house_size\": 320,\n",
      "    \"listings_description\": \"Step back in time with this historic 5-bedroom, 4-bathroom home in the Heritage District. Featuring elegant architecture, a grand staircase, and a beautifully landscaped garden, this property exudes timeless charm.\",\n",
      "    \"listings_neighborhood_description\": \"The Heritage District is a historic neighborhood filled with tree-lined streets, heritage homes, and boutique shops. Immerse yourself in the rich history and architectural beauty of this unique district.\"\n",
      "},\n",
      "{\n",
      "    \"listings_headline\": \"Contemporary Living in Skyline Heights\",\n",
      "    \"listings_neighborhood\": \"Skyline Heights\",\n",
      "    \"listings_price\": 2800000,\n",
      "    \"listings_bedrooms\": 6,\n",
      "    \"listings_bathrooms\": 5,\n",
      "    \"listings_house_size\": 400,\n",
      "    \"listings_description\": \"Experience contemporary living at its finest in this 6-bedroom, 5-bathroom home in Skyline Heights. With sleek design, high ceilings, and a rooftop terrace with city views, this property is perfect for modern urban dwellers.\",\n",
      "    \"listings_neighborhood_description\": \"Skyline Heights offers a vibrant urban lifestyle with trendy cafes, art galleries, and easy access to public transportation. Enjoy the bustling energy of the city from the comfort of your stylish home.\"\n",
      "},\n",
      "{\n",
      "    \"listings_headline\": \"Tranquil Haven in Willow Creek\",\n",
      "    \"listings_neighborhood\": \"Willow Creek\",\n",
      "    \"listings_price\": 1500000,\n",
      "    \"listings_bedrooms\": 4,\n",
      "    \"listings_bathrooms\": 3,\n",
      "    \"listings_house_size\": 250,\n",
      "    \"listings_description\": \"Find peace and serenity in this tranquil 4-bedroom, 3-bathroom home in Willow Creek. Surrounded by lush greenery, with a private backyard and a cozy fireplace, this property offers a perfect retreat from the hustle and bustle of city life.\",\n",
      "    \"listings_neighborhood_description\": \"Willow Creek is a quiet residential neighborhood known for its tree-lined streets, parks, and family-friendly atmosphere. Enjoy a sense of community and nature in this peaceful enclave.\"\n",
      "},\n",
      "{\n",
      "    \"listings_headline\": \"Urban Chic in Downtown District\",\n",
      "    \"listings_neighborhood\": \"Downtown District\",\n",
      "    \"listings_price\": 2000000,\n",
      "    \"listings_bedrooms\": 3,\n",
      "    \"listings_bathrooms\": 2,\n",
      "    \"listings_house_size\": 180,\n",
      "    \"listings_description\": \"Embrace urban chic living in this stylish 3-bedroom, 2-bathroom home in the heart of the Downtown District. With modern design, high-end appliances, and a rooftop patio for entertaining, this property is perfect for those who love city living.\",\n",
      "    \"listings_neighborhood_description\": \"The Downtown District is a bustling urban center with a vibrant nightlife, trendy restaurants, and cultural attractions. Experience the excitement of city life right outside your doorstep.\"\n",
      "},\n",
      "{\n",
      "    \"listings_headline\": \"Rustic Charm in Maple Grove\",\n",
      "    \"listings_neighborhood\": \"Maple Grove\",\n",
      "    \"listings_price\": 1300000,\n",
      "    \"listings_bedrooms\": 4,\n",
      "    \"listings_bathrooms\": 3,\n",
      "    \"listings_house_size\": 220,\n",
      "    \"listings_description\": \"Discover the rustic charm of this 4-bedroom, 3-bathroom home in Maple Grove. With exposed wooden beams, a cozy fireplace, and a spacious backyard with a fire pit, this property offers a warm and inviting atmosphere.\",\n",
      "    \"listings_neighborhood_description\": \"Maple Grove is a quaint neighborhood with tree-lined streets, local cafes, and boutique shops. Enjoy a sense of community and small-town charm in this cozy enclave.\"\n",
      "},\n",
      "{\n",
      "    \"listings_headline\": \"Luxurious Retreat in Hillside Manor\",\n",
      "    \"listings_neighborhood\": \"Hillside Manor\",\n",
      "    \"listings_price\": 3000000,\n",
      "    \"listings_bedrooms\": 5,\n",
      "    \"listings_bathrooms\": 4,\n",
      "    \"listings_house_size\": 350,\n",
      "    \"listings_description\": \"Escape to a luxurious retreat in the prestigious Hillside Manor. This 5-bedroom, 4-bathroom home features a gourmet kitchen, a home theater, and a landscaped garden with a pool, offering the ultimate in upscale living.\",\n",
      "    \"listings_neighborhood_description\": \"Hillside Manor is an exclusive neighborhood known for its luxury estates, private golf courses, and scenic views. Enjoy privacy and sophistication in this upscale enclave.\"\n",
      "},\n",
      "{\n",
      "    \"listings_headline\": \"Coastal Living in Seaview Heights\",\n",
      "    \"listings_neighborhood\": \"Seaview Heights\",\n",
      "    \"listings_price\": 2200000,\n",
      "    \"listings_bedrooms\": 3,\n",
      "    \"listings_bathrooms\": 2,\n",
      "    \"listings_house_size\": 190,\n",
      "    \"listings_description\": \"Experience the beauty of coastal living in this 3-bedroom, 2-bathroom home in Seaview Heights. With ocean views, a sun-drenched deck, and beach access just steps away, this property offers a perfect seaside retreat.\",\n",
      "    \"listings_neighborhood_description\": \"Seaview Heights is a coastal community with sandy beaches, seaside cafes, and water sports activities. Enjoy the laid-back lifestyle and stunning ocean views in this charming neighborhood.\"\n",
      "},\n",
      "{\n",
      "    \"listings_headline\": \"Elegant Oasis in Rosewood Estates\",\n",
      "    \"listings_neighborhood\": \"Rosewood Estates\",\n",
      "    \"listings_price\": 2600000,\n",
      "    \"listings_bedrooms\": 4,\n",
      "    \"listings_bathrooms\": 3,\n",
      "    \"listings_house_size\": 240,\n",
      "    \"listings_description\": \"Discover an elegant oasis in the prestigious Rosewood Estates. This 4-bedroom, 3-bathroom home features a grand foyer, a gourmet kitchen, and a landscaped backyard with a fountain, offering a perfect blend of luxury and sophistication.\",\n",
      "    \"listings_neighborhood_description\": \"Rosewood Estates is an upscale neighborhood known for its luxury homes, private estates, and exclusive country clubs. Experience the epitome of luxury living in this prestigious enclave.\"\n",
      "},\n",
      "{\n",
      "    \"listings_headline\": \"Serenity in Pinecrest Meadows\",\n",
      "    \"listings_neighborhood\": \"Pinecrest Meadows\",\n",
      "    \"listings_price\": 1700000,\n",
      "    \"listings_bedrooms\": 5,\n",
      "    \"listings_bathrooms\": 4,\n",
      "    \"listings_house_size\": 260,\n",
      "    \"listings_description\": \"Find serenity in this 5-bedroom, 4-bathroom home in Pinecrest Meadows. Surrounded by towering pine trees, with a wrap-around porch and a cozy fireplace, this property offers a peaceful retreat in a natural setting.\",\n",
      "    \"listings_neighborhood_description\": \"Pinecrest Meadows is a nature lover's paradise with hiking trails, wildlife, and scenic views. Enjoy the tranquility and beauty of the outdoors in this idyllic neighborhood.\"\n",
      "},\n",
      "{\n",
      "    \"listings_headline\": \"Modern Elegance in Silver Lake\",\n",
      "    \"listings_neighborhood\": \"Silver Lake\",\n",
      "    \"listings_price\": 1900000,\n",
      "    \"listings_bedrooms\": 3,\n",
      "    \"listings_bathrooms\": 2,\n",
      "    \"listings_house_size\": 210,\n",
      "    \"listings_description\": \"Experience modern elegance in this 3-bedroom, 2-bathroom home in Silver Lake. With sleek design, high ceilings, and a rooftop deck with panoramic views of the lake, this property offers a perfect blend of style and sophistication.\",\n",
      "    \"listings_neighborhood_description\": \"Silver Lake is a trendy neighborhood known for its hip cafes, art galleries, and scenic lake views. Embrace the vibrant energy and creative spirit of this eclectic community.\"\n",
      "},\n",
      "{\n",
      "    \"listings_headline\": \"Charming Cottage in Willowbrook Village\",\n",
      "    \"listings_neighborhood\": \"Willowbrook Village\",\n",
      "    \"listings_price\": 1400000,\n",
      "    \"listings_bedrooms\": 2,\n",
      "    \"listings_bathrooms\": 1,\n",
      "    \"listings_house_size\": 150,\n",
      "    \"listings_description\": \"Escape to this charming cottage in the heart of Willowbrook Village. With 2 bedrooms, 1 bathroom, a cozy fireplace, and a flower-filled garden, this property offers a perfect retreat for those seeking a peaceful lifestyle.\",\n",
      "    \"listings_neighborhood_description\": \"Willowbrook Village is a quaint village with historic cottages, local bakeries, and community events. Enjoy a sense of nostalgia and small-town charm in this idyllic setting.\"\n",
      "},\n",
      "{\n",
      "    \"listings_headline\": \"Luxury Living in Diamond Heights\",\n",
      "    \"listings_neighborhood\": \"Diamond Heights\",\n",
      "    \"listings_price\": 2700000,\n",
      "    \"listings_bedrooms\": 4,\n",
      "    \"listings_bathrooms\": 3,\n",
      "    \"listings_house_size\": 230,\n",
      "    \"listings_description\": \"Indulge in luxury living in this 4-bedroom, 3-bathroom home in Diamond Heights. With high-end finishes, a gourmet kitchen, and a landscaped backyard with a pool, this property offers the ultimate in upscale living.\",\n",
      "    \"listings_neighborhood_description\": \"Diamond Heights is an exclusive neighborhood known for its luxury estates, private tennis clubs, and panoramic views. Experience privacy and sophistication in this prestigious enclave.\"\n",
      "},\n",
      "{\n",
      "    \"listings_headline\": \"Tranquil Retreat in Cedar Valley\",\n",
      "    \"listings_neighborhood\": \"Cedar Valley\",\n",
      "    \"listings_price\": 1600000,\n",
      "    \"listings_bedrooms\": 3,\n",
      "    \"listings_bathrooms\": 2,\n",
      "    \"listings_house_size\": 170,\n",
      "    \"listings_description\": \"Find peace and tranquility in this 3-bedroom, 2-bathroom home in Cedar Valley. Surrounded by cedar trees, with a wrap-around porch and a meditation garden, this property offers a serene retreat in a natural setting.\",\n",
      "    \"listings_neighborhood_description\": \"Cedar Valley is a peaceful neighborhood with hiking trails, nature reserves, and a strong sense of community. Enjoy the beauty of the outdoors and a slower pace of life in this idyllic enclave.\"\n",
      "},\n",
      "{\n",
      "    \"listings_headline\": \"Urban Oasis in City Center\",\n",
      "    \"listings_neighborhood\": \"City Center\",\n",
      "    \"listings_price\": 2500000,\n",
      "    \"listings_bedrooms\": 4,\n",
      "    \"listings_bathrooms\": 3,\n",
      "    \"listings_house_size\": 260,\n",
      "    \"listings_description\": \"Discover an urban oasis in the heart of the City Center. This 4-bedroom, 3-bathroom home features modern design, a rooftop terrace with city views, and a home office, offering a perfect blend of style and functionality.\",\n",
      "    \"listings_neighborhood_description\": \"City Center is a vibrant urban hub with shopping centers, restaurants, and cultural attractions. Experience the convenience and excitement of city living in this bustling neighborhood.\"\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(response.choices[0].message['content'][:-2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "53a1df75",
   "metadata": {},
   "outputs": [],
   "source": [
    "property_listings = {}\n",
    "for i, desc in enumerate(eval(response.choices[0].message['content'][:-2].replace('\\n', ''))):\n",
    "    property_listings[i] = desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "cfb2bcc2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mProperty 0 : Modern Luxury in Central Park\u001b[0m\n",
      "- neighborhood              : Central Park\n",
      "- price                     : 3500000\n",
      "- bedrooms                  : 4\n",
      "- bathrooms                 : 3\n",
      "- house size                : 280\n",
      "- description               : Indulge in the epitome of modern luxury with this stunning 4-bedroom, 3-bathroom home in Central Park. Boasting high-end finishes, a gourmet kitchen, and panoramic views of the city skyline, this property offers a sophisticated urban lifestyle.\n",
      "- neighborhood description  : Central Park is a vibrant neighborhood known for its upscale dining, shopping, and cultural attractions. Enjoy easy access to the park for leisurely strolls and outdoor activities.\n",
      "-------------------------------------------------\n",
      "\u001b[33mProperty 1 : Seaside Retreat in Ocean View\u001b[0m\n",
      "- neighborhood              : Ocean View\n",
      "- price                     : 1800000\n",
      "- bedrooms                  : 3\n",
      "- bathrooms                 : 2\n",
      "- house size                : 200\n",
      "- description               : Escape to this charming seaside retreat in Ocean View. With 3 bedrooms, 2 bathrooms, and a spacious deck overlooking the ocean, this home offers a perfect blend of coastal living and modern comfort.\n",
      "- neighborhood description  : Ocean View is a picturesque coastal community with sandy beaches, seafood restaurants, and a laid-back atmosphere. Experience the tranquility of seaside living in this charming neighborhood.\n",
      "-------------------------------------------------\n",
      "\u001b[33mProperty 2 : Historic Elegance in Heritage District\u001b[0m\n",
      "- neighborhood              : Heritage District\n",
      "- price                     : 1200000\n",
      "- bedrooms                  : 5\n",
      "- bathrooms                 : 4\n",
      "- house size                : 320\n",
      "- description               : Step back in time with this historic 5-bedroom, 4-bathroom home in the Heritage District. Featuring elegant architecture, a grand staircase, and a beautifully landscaped garden, this property exudes timeless charm.\n",
      "- neighborhood description  : The Heritage District is a historic neighborhood filled with tree-lined streets, heritage homes, and boutique shops. Immerse yourself in the rich history and architectural beauty of this unique district.\n",
      "-------------------------------------------------\n",
      "\u001b[33mProperty 3 : Contemporary Living in Skyline Heights\u001b[0m\n",
      "- neighborhood              : Skyline Heights\n",
      "- price                     : 2800000\n",
      "- bedrooms                  : 6\n",
      "- bathrooms                 : 5\n",
      "- house size                : 400\n",
      "- description               : Experience contemporary living at its finest in this 6-bedroom, 5-bathroom home in Skyline Heights. With sleek design, high ceilings, and a rooftop terrace with city views, this property is perfect for modern urban dwellers.\n",
      "- neighborhood description  : Skyline Heights offers a vibrant urban lifestyle with trendy cafes, art galleries, and easy access to public transportation. Enjoy the bustling energy of the city from the comfort of your stylish home.\n",
      "-------------------------------------------------\n",
      "\u001b[33mProperty 4 : Tranquil Haven in Willow Creek\u001b[0m\n",
      "- neighborhood              : Willow Creek\n",
      "- price                     : 1500000\n",
      "- bedrooms                  : 4\n",
      "- bathrooms                 : 3\n",
      "- house size                : 250\n",
      "- description               : Find peace and serenity in this tranquil 4-bedroom, 3-bathroom home in Willow Creek. Surrounded by lush greenery, with a private backyard and a cozy fireplace, this property offers a perfect retreat from the hustle and bustle of city life.\n",
      "- neighborhood description  : Willow Creek is a quiet residential neighborhood known for its tree-lined streets, parks, and family-friendly atmosphere. Enjoy a sense of community and nature in this peaceful enclave.\n",
      "-------------------------------------------------\n",
      "\u001b[33mProperty 5 : Urban Chic in Downtown District\u001b[0m\n",
      "- neighborhood              : Downtown District\n",
      "- price                     : 2000000\n",
      "- bedrooms                  : 3\n",
      "- bathrooms                 : 2\n",
      "- house size                : 180\n",
      "- description               : Embrace urban chic living in this stylish 3-bedroom, 2-bathroom home in the heart of the Downtown District. With modern design, high-end appliances, and a rooftop patio for entertaining, this property is perfect for those who love city living.\n",
      "- neighborhood description  : The Downtown District is a bustling urban center with a vibrant nightlife, trendy restaurants, and cultural attractions. Experience the excitement of city life right outside your doorstep.\n",
      "-------------------------------------------------\n",
      "\u001b[33mProperty 6 : Rustic Charm in Maple Grove\u001b[0m\n",
      "- neighborhood              : Maple Grove\n",
      "- price                     : 1300000\n",
      "- bedrooms                  : 4\n",
      "- bathrooms                 : 3\n",
      "- house size                : 220\n",
      "- description               : Discover the rustic charm of this 4-bedroom, 3-bathroom home in Maple Grove. With exposed wooden beams, a cozy fireplace, and a spacious backyard with a fire pit, this property offers a warm and inviting atmosphere.\n",
      "- neighborhood description  : Maple Grove is a quaint neighborhood with tree-lined streets, local cafes, and boutique shops. Enjoy a sense of community and small-town charm in this cozy enclave.\n",
      "-------------------------------------------------\n",
      "\u001b[33mProperty 7 : Luxurious Retreat in Hillside Manor\u001b[0m\n",
      "- neighborhood              : Hillside Manor\n",
      "- price                     : 3000000\n",
      "- bedrooms                  : 5\n",
      "- bathrooms                 : 4\n",
      "- house size                : 350\n",
      "- description               : Escape to a luxurious retreat in the prestigious Hillside Manor. This 5-bedroom, 4-bathroom home features a gourmet kitchen, a home theater, and a landscaped garden with a pool, offering the ultimate in upscale living.\n",
      "- neighborhood description  : Hillside Manor is an exclusive neighborhood known for its luxury estates, private golf courses, and scenic views. Enjoy privacy and sophistication in this upscale enclave.\n",
      "-------------------------------------------------\n",
      "\u001b[33mProperty 8 : Coastal Living in Seaview Heights\u001b[0m\n",
      "- neighborhood              : Seaview Heights\n",
      "- price                     : 2200000\n",
      "- bedrooms                  : 3\n",
      "- bathrooms                 : 2\n",
      "- house size                : 190\n",
      "- description               : Experience the beauty of coastal living in this 3-bedroom, 2-bathroom home in Seaview Heights. With ocean views, a sun-drenched deck, and beach access just steps away, this property offers a perfect seaside retreat.\n",
      "- neighborhood description  : Seaview Heights is a coastal community with sandy beaches, seaside cafes, and water sports activities. Enjoy the laid-back lifestyle and stunning ocean views in this charming neighborhood.\n",
      "-------------------------------------------------\n",
      "\u001b[33mProperty 9 : Elegant Oasis in Rosewood Estates\u001b[0m\n",
      "- neighborhood              : Rosewood Estates\n",
      "- price                     : 2600000\n",
      "- bedrooms                  : 4\n",
      "- bathrooms                 : 3\n",
      "- house size                : 240\n",
      "- description               : Discover an elegant oasis in the prestigious Rosewood Estates. This 4-bedroom, 3-bathroom home features a grand foyer, a gourmet kitchen, and a landscaped backyard with a fountain, offering a perfect blend of luxury and sophistication.\n",
      "- neighborhood description  : Rosewood Estates is an upscale neighborhood known for its luxury homes, private estates, and exclusive country clubs. Experience the epitome of luxury living in this prestigious enclave.\n",
      "-------------------------------------------------\n",
      "\u001b[33mProperty 10 : Serenity in Pinecrest Meadows\u001b[0m\n",
      "- neighborhood              : Pinecrest Meadows\n",
      "- price                     : 1700000\n",
      "- bedrooms                  : 5\n",
      "- bathrooms                 : 4\n",
      "- house size                : 260\n",
      "- description               : Find serenity in this 5-bedroom, 4-bathroom home in Pinecrest Meadows. Surrounded by towering pine trees, with a wrap-around porch and a cozy fireplace, this property offers a peaceful retreat in a natural setting.\n",
      "- neighborhood description  : Pinecrest Meadows is a nature lover's paradise with hiking trails, wildlife, and scenic views. Enjoy the tranquility and beauty of the outdoors in this idyllic neighborhood.\n",
      "-------------------------------------------------\n",
      "\u001b[33mProperty 11 : Modern Elegance in Silver Lake\u001b[0m\n",
      "- neighborhood              : Silver Lake\n",
      "- price                     : 1900000\n",
      "- bedrooms                  : 3\n",
      "- bathrooms                 : 2\n",
      "- house size                : 210\n",
      "- description               : Experience modern elegance in this 3-bedroom, 2-bathroom home in Silver Lake. With sleek design, high ceilings, and a rooftop deck with panoramic views of the lake, this property offers a perfect blend of style and sophistication.\n",
      "- neighborhood description  : Silver Lake is a trendy neighborhood known for its hip cafes, art galleries, and scenic lake views. Embrace the vibrant energy and creative spirit of this eclectic community.\n",
      "-------------------------------------------------\n",
      "\u001b[33mProperty 12 : Charming Cottage in Willowbrook Village\u001b[0m\n",
      "- neighborhood              : Willowbrook Village\n",
      "- price                     : 1400000\n",
      "- bedrooms                  : 2\n",
      "- bathrooms                 : 1\n",
      "- house size                : 150\n",
      "- description               : Escape to this charming cottage in the heart of Willowbrook Village. With 2 bedrooms, 1 bathroom, a cozy fireplace, and a flower-filled garden, this property offers a perfect retreat for those seeking a peaceful lifestyle.\n",
      "- neighborhood description  : Willowbrook Village is a quaint village with historic cottages, local bakeries, and community events. Enjoy a sense of nostalgia and small-town charm in this idyllic setting.\n",
      "-------------------------------------------------\n",
      "\u001b[33mProperty 13 : Luxury Living in Diamond Heights\u001b[0m\n",
      "- neighborhood              : Diamond Heights\n",
      "- price                     : 2700000\n",
      "- bedrooms                  : 4\n",
      "- bathrooms                 : 3\n",
      "- house size                : 230\n",
      "- description               : Indulge in luxury living in this 4-bedroom, 3-bathroom home in Diamond Heights. With high-end finishes, a gourmet kitchen, and a landscaped backyard with a pool, this property offers the ultimate in upscale living.\n",
      "- neighborhood description  : Diamond Heights is an exclusive neighborhood known for its luxury estates, private tennis clubs, and panoramic views. Experience privacy and sophistication in this prestigious enclave.\n",
      "-------------------------------------------------\n",
      "\u001b[33mProperty 14 : Tranquil Retreat in Cedar Valley\u001b[0m\n",
      "- neighborhood              : Cedar Valley\n",
      "- price                     : 1600000\n",
      "- bedrooms                  : 3\n",
      "- bathrooms                 : 2\n",
      "- house size                : 170\n",
      "- description               : Find peace and tranquility in this 3-bedroom, 2-bathroom home in Cedar Valley. Surrounded by cedar trees, with a wrap-around porch and a meditation garden, this property offers a serene retreat in a natural setting.\n",
      "- neighborhood description  : Cedar Valley is a peaceful neighborhood with hiking trails, nature reserves, and a strong sense of community. Enjoy the beauty of the outdoors and a slower pace of life in this idyllic enclave.\n",
      "-------------------------------------------------\n",
      "\u001b[33mProperty 15 : Urban Oasis in City Center\u001b[0m\n",
      "- neighborhood              : City Center\n",
      "- price                     : 2500000\n",
      "- bedrooms                  : 4\n",
      "- bathrooms                 : 3\n",
      "- house size                : 260\n",
      "- description               : Discover an urban oasis in the heart of the City Center. This 4-bedroom, 3-bathroom home features modern design, a rooftop terrace with city views, and a home office, offering a perfect blend of style and functionality.\n",
      "- neighborhood description  : City Center is a vibrant urban hub with shopping centers, restaurants, and cultural attractions. Experience the convenience and excitement of city living in this bustling neighborhood.\n",
      "-------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "for i, listing in property_listings.items():\n",
    "    print(YELLOW + f\"Property {str(i)} : {listing['listings_headline']}\" + ENDC)\n",
    "    for key in list(listings_description_model.model_fields.keys())[1:]:\n",
    "        print(f'- {key.split(\"listings_\")[1].replace(\"_\",\" \"):<25} : {listing[key]}')\n",
    "    print('-------------------------------------------------')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1fbddb0",
   "metadata": {},
   "source": [
    "- save the synthetic listing descriptions\n",
    "- we concatenate the descriptions in one column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "id": "b87aa21b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>listings_headline</th>\n",
       "      <th>listings_neighborhood</th>\n",
       "      <th>listings_price</th>\n",
       "      <th>listings_bedrooms</th>\n",
       "      <th>listings_bathrooms</th>\n",
       "      <th>listings_house_size</th>\n",
       "      <th>listings_description</th>\n",
       "      <th>listings_neighborhood_description</th>\n",
       "      <th>full_description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Modern Luxury in Central Park</td>\n",
       "      <td>Central Park</td>\n",
       "      <td>3500000</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>280</td>\n",
       "      <td>Indulge in the epitome of modern luxury with t...</td>\n",
       "      <td>Central Park is a vibrant neighborhood known f...</td>\n",
       "      <td>Modern Luxury in Central Park: Indulge in the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Seaside Retreat in Ocean View</td>\n",
       "      <td>Ocean View</td>\n",
       "      <td>1800000</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>200</td>\n",
       "      <td>Escape to this charming seaside retreat in Oce...</td>\n",
       "      <td>Ocean View is a picturesque coastal community ...</td>\n",
       "      <td>Seaside Retreat in Ocean View: Escape to this ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Historic Elegance in Heritage District</td>\n",
       "      <td>Heritage District</td>\n",
       "      <td>1200000</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>320</td>\n",
       "      <td>Step back in time with this historic 5-bedroom...</td>\n",
       "      <td>The Heritage District is a historic neighborho...</td>\n",
       "      <td>Historic Elegance in Heritage District: Step b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Contemporary Living in Skyline Heights</td>\n",
       "      <td>Skyline Heights</td>\n",
       "      <td>2800000</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>400</td>\n",
       "      <td>Experience contemporary living at its finest i...</td>\n",
       "      <td>Skyline Heights offers a vibrant urban lifesty...</td>\n",
       "      <td>Contemporary Living in Skyline Heights: Experi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Tranquil Haven in Willow Creek</td>\n",
       "      <td>Willow Creek</td>\n",
       "      <td>1500000</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>250</td>\n",
       "      <td>Find peace and serenity in this tranquil 4-bed...</td>\n",
       "      <td>Willow Creek is a quiet residential neighborho...</td>\n",
       "      <td>Tranquil Haven in Willow Creek: Find peace and...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Urban Chic in Downtown District</td>\n",
       "      <td>Downtown District</td>\n",
       "      <td>2000000</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>180</td>\n",
       "      <td>Embrace urban chic living in this stylish 3-be...</td>\n",
       "      <td>The Downtown District is a bustling urban cent...</td>\n",
       "      <td>Urban Chic in Downtown District: Embrace urban...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Rustic Charm in Maple Grove</td>\n",
       "      <td>Maple Grove</td>\n",
       "      <td>1300000</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>220</td>\n",
       "      <td>Discover the rustic charm of this 4-bedroom, 3...</td>\n",
       "      <td>Maple Grove is a quaint neighborhood with tree...</td>\n",
       "      <td>Rustic Charm in Maple Grove: Discover the rust...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Luxurious Retreat in Hillside Manor</td>\n",
       "      <td>Hillside Manor</td>\n",
       "      <td>3000000</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>350</td>\n",
       "      <td>Escape to a luxurious retreat in the prestigio...</td>\n",
       "      <td>Hillside Manor is an exclusive neighborhood kn...</td>\n",
       "      <td>Luxurious Retreat in Hillside Manor: Escape to...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Coastal Living in Seaview Heights</td>\n",
       "      <td>Seaview Heights</td>\n",
       "      <td>2200000</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>190</td>\n",
       "      <td>Experience the beauty of coastal living in thi...</td>\n",
       "      <td>Seaview Heights is a coastal community with sa...</td>\n",
       "      <td>Coastal Living in Seaview Heights: Experience ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Elegant Oasis in Rosewood Estates</td>\n",
       "      <td>Rosewood Estates</td>\n",
       "      <td>2600000</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>240</td>\n",
       "      <td>Discover an elegant oasis in the prestigious R...</td>\n",
       "      <td>Rosewood Estates is an upscale neighborhood kn...</td>\n",
       "      <td>Elegant Oasis in Rosewood Estates: Discover an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Serenity in Pinecrest Meadows</td>\n",
       "      <td>Pinecrest Meadows</td>\n",
       "      <td>1700000</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>260</td>\n",
       "      <td>Find serenity in this 5-bedroom, 4-bathroom ho...</td>\n",
       "      <td>Pinecrest Meadows is a nature lover's paradise...</td>\n",
       "      <td>Serenity in Pinecrest Meadows: Find serenity i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Modern Elegance in Silver Lake</td>\n",
       "      <td>Silver Lake</td>\n",
       "      <td>1900000</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>210</td>\n",
       "      <td>Experience modern elegance in this 3-bedroom, ...</td>\n",
       "      <td>Silver Lake is a trendy neighborhood known for...</td>\n",
       "      <td>Modern Elegance in Silver Lake: Experience mod...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Charming Cottage in Willowbrook Village</td>\n",
       "      <td>Willowbrook Village</td>\n",
       "      <td>1400000</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>150</td>\n",
       "      <td>Escape to this charming cottage in the heart o...</td>\n",
       "      <td>Willowbrook Village is a quaint village with h...</td>\n",
       "      <td>Charming Cottage in Willowbrook Village: Escap...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Luxury Living in Diamond Heights</td>\n",
       "      <td>Diamond Heights</td>\n",
       "      <td>2700000</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>230</td>\n",
       "      <td>Indulge in luxury living in this 4-bedroom, 3-...</td>\n",
       "      <td>Diamond Heights is an exclusive neighborhood k...</td>\n",
       "      <td>Luxury Living in Diamond Heights: Indulge in l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Tranquil Retreat in Cedar Valley</td>\n",
       "      <td>Cedar Valley</td>\n",
       "      <td>1600000</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>170</td>\n",
       "      <td>Find peace and tranquility in this 3-bedroom, ...</td>\n",
       "      <td>Cedar Valley is a peaceful neighborhood with h...</td>\n",
       "      <td>Tranquil Retreat in Cedar Valley: Find peace a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Urban Oasis in City Center</td>\n",
       "      <td>City Center</td>\n",
       "      <td>2500000</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>260</td>\n",
       "      <td>Discover an urban oasis in the heart of the Ci...</td>\n",
       "      <td>City Center is a vibrant urban hub with shoppi...</td>\n",
       "      <td>Urban Oasis in City Center: Discover an urban ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          listings_headline listings_neighborhood  \\\n",
       "0             Modern Luxury in Central Park          Central Park   \n",
       "1             Seaside Retreat in Ocean View            Ocean View   \n",
       "2    Historic Elegance in Heritage District     Heritage District   \n",
       "3    Contemporary Living in Skyline Heights       Skyline Heights   \n",
       "4            Tranquil Haven in Willow Creek          Willow Creek   \n",
       "5           Urban Chic in Downtown District     Downtown District   \n",
       "6               Rustic Charm in Maple Grove           Maple Grove   \n",
       "7       Luxurious Retreat in Hillside Manor        Hillside Manor   \n",
       "8         Coastal Living in Seaview Heights       Seaview Heights   \n",
       "9         Elegant Oasis in Rosewood Estates      Rosewood Estates   \n",
       "10            Serenity in Pinecrest Meadows     Pinecrest Meadows   \n",
       "11           Modern Elegance in Silver Lake           Silver Lake   \n",
       "12  Charming Cottage in Willowbrook Village   Willowbrook Village   \n",
       "13         Luxury Living in Diamond Heights       Diamond Heights   \n",
       "14         Tranquil Retreat in Cedar Valley          Cedar Valley   \n",
       "15               Urban Oasis in City Center           City Center   \n",
       "\n",
       "   listings_price listings_bedrooms listings_bathrooms listings_house_size  \\\n",
       "0         3500000                 4                  3                 280   \n",
       "1         1800000                 3                  2                 200   \n",
       "2         1200000                 5                  4                 320   \n",
       "3         2800000                 6                  5                 400   \n",
       "4         1500000                 4                  3                 250   \n",
       "5         2000000                 3                  2                 180   \n",
       "6         1300000                 4                  3                 220   \n",
       "7         3000000                 5                  4                 350   \n",
       "8         2200000                 3                  2                 190   \n",
       "9         2600000                 4                  3                 240   \n",
       "10        1700000                 5                  4                 260   \n",
       "11        1900000                 3                  2                 210   \n",
       "12        1400000                 2                  1                 150   \n",
       "13        2700000                 4                  3                 230   \n",
       "14        1600000                 3                  2                 170   \n",
       "15        2500000                 4                  3                 260   \n",
       "\n",
       "                                 listings_description  \\\n",
       "0   Indulge in the epitome of modern luxury with t...   \n",
       "1   Escape to this charming seaside retreat in Oce...   \n",
       "2   Step back in time with this historic 5-bedroom...   \n",
       "3   Experience contemporary living at its finest i...   \n",
       "4   Find peace and serenity in this tranquil 4-bed...   \n",
       "5   Embrace urban chic living in this stylish 3-be...   \n",
       "6   Discover the rustic charm of this 4-bedroom, 3...   \n",
       "7   Escape to a luxurious retreat in the prestigio...   \n",
       "8   Experience the beauty of coastal living in thi...   \n",
       "9   Discover an elegant oasis in the prestigious R...   \n",
       "10  Find serenity in this 5-bedroom, 4-bathroom ho...   \n",
       "11  Experience modern elegance in this 3-bedroom, ...   \n",
       "12  Escape to this charming cottage in the heart o...   \n",
       "13  Indulge in luxury living in this 4-bedroom, 3-...   \n",
       "14  Find peace and tranquility in this 3-bedroom, ...   \n",
       "15  Discover an urban oasis in the heart of the Ci...   \n",
       "\n",
       "                    listings_neighborhood_description  \\\n",
       "0   Central Park is a vibrant neighborhood known f...   \n",
       "1   Ocean View is a picturesque coastal community ...   \n",
       "2   The Heritage District is a historic neighborho...   \n",
       "3   Skyline Heights offers a vibrant urban lifesty...   \n",
       "4   Willow Creek is a quiet residential neighborho...   \n",
       "5   The Downtown District is a bustling urban cent...   \n",
       "6   Maple Grove is a quaint neighborhood with tree...   \n",
       "7   Hillside Manor is an exclusive neighborhood kn...   \n",
       "8   Seaview Heights is a coastal community with sa...   \n",
       "9   Rosewood Estates is an upscale neighborhood kn...   \n",
       "10  Pinecrest Meadows is a nature lover's paradise...   \n",
       "11  Silver Lake is a trendy neighborhood known for...   \n",
       "12  Willowbrook Village is a quaint village with h...   \n",
       "13  Diamond Heights is an exclusive neighborhood k...   \n",
       "14  Cedar Valley is a peaceful neighborhood with h...   \n",
       "15  City Center is a vibrant urban hub with shoppi...   \n",
       "\n",
       "                                     full_description  \n",
       "0   Modern Luxury in Central Park: Indulge in the ...  \n",
       "1   Seaside Retreat in Ocean View: Escape to this ...  \n",
       "2   Historic Elegance in Heritage District: Step b...  \n",
       "3   Contemporary Living in Skyline Heights: Experi...  \n",
       "4   Tranquil Haven in Willow Creek: Find peace and...  \n",
       "5   Urban Chic in Downtown District: Embrace urban...  \n",
       "6   Rustic Charm in Maple Grove: Discover the rust...  \n",
       "7   Luxurious Retreat in Hillside Manor: Escape to...  \n",
       "8   Coastal Living in Seaview Heights: Experience ...  \n",
       "9   Elegant Oasis in Rosewood Estates: Discover an...  \n",
       "10  Serenity in Pinecrest Meadows: Find serenity i...  \n",
       "11  Modern Elegance in Silver Lake: Experience mod...  \n",
       "12  Charming Cottage in Willowbrook Village: Escap...  \n",
       "13  Luxury Living in Diamond Heights: Indulge in l...  \n",
       "14  Tranquil Retreat in Cedar Valley: Find peace a...  \n",
       "15  Urban Oasis in City Center: Discover an urban ...  "
      ]
     },
     "execution_count": 296,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(property_listings).T\n",
    "df['full_description'] = df.apply(lambda x: x['listings_headline'] + ': ' + x['listings_description'] + ' Neighborhood: ' + x['listings_neighborhood_description'], axis=1)\n",
    "df.to_csv('data/property_listings.csv', index=False, header=True)\n",
    "df.to_json('data/property_listings.json', orient='records')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 519,
   "id": "1a96e661",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export DataFrame to text file\n",
    "'''with open('./data/listings.txt', 'w') as f:\n",
    "    df_string = df.to_string(header=True, index=True)\n",
    "    f.write(df_string)'''"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f1223d4",
   "metadata": {},
   "source": [
    "# STEP 2 - POPULATE VECTORSTORE WITH SYNTHETIC LISTINGS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "id": "513c31bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "'''loader = CSVLoader(file_path='data/property_listings.csv') #, metadata_columns=df.columns.tolist()\n",
    "docs = loader.load()\n",
    "print(f'Number of properties: {len(docs)}')\n",
    "print(docs[0].page_content)'''\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef369612",
   "metadata": {},
   "source": [
    "PREPARE DATA FOR EMBEDDING STEP\n",
    "- JSONLoader from langchain requires jq package not available on windows.\n",
    "- We define a specific JSONLoader to capture the metadata of the JSON file\n",
    "- code from here: https://github.com/langchain-ai/langchain/issues/4396"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fdb52d84",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from pathlib import Path\n",
    "from typing import Callable, Dict, List, Optional, Union\n",
    "\n",
    "from langchain.docstore.document import Document\n",
    "from langchain.document_loaders.base import BaseLoader\n",
    "\n",
    "class JSONLoader(BaseLoader):\n",
    "    def __init__(\n",
    "        self,\n",
    "        file_path: Union[str, Path],\n",
    "        content_key: Optional[str] = None,\n",
    "        metadata_func: Optional[Callable[[dict, dict], dict]] = None,\n",
    "    ):\n",
    "        \"\"\"\n",
    "        Initializes the JSONLoader with a file path, an optional content key to extract specific content,\n",
    "        and an optional metadata function to extract metadata from each record.\n",
    "        \"\"\"\n",
    "        self.file_path = Path(file_path).resolve()\n",
    "        self.content_key = content_key \n",
    "        self.metadata_func = metadata_func\n",
    "\n",
    "    def create_documents(self, processed_data):\n",
    "        \"\"\"\n",
    "        Creates Document objects from processed data.\n",
    "        \"\"\"\n",
    "        documents = []\n",
    "        for item in processed_data:\n",
    "            content = item.get('content', '')  \n",
    "            metadata = item.get('metadata', {})\n",
    "            document = Document(page_content=content, metadata=metadata)\n",
    "            documents.append(document)\n",
    "        return documents\n",
    "\n",
    "    def process_json(self, data):\n",
    "        \"\"\"\n",
    "        Processes JSON data to prepare for document creation, extracting content based on the content_key\n",
    "        and applying the metadata function if provided.\n",
    "        \"\"\"\n",
    "        processed_data = []\n",
    "        if isinstance(data, list):\n",
    "            for item in data:\n",
    "                content = item.get(self.content_key, '') if self.content_key else ''\n",
    "                metadata = {}\n",
    "                if self.metadata_func and isinstance(item, dict):\n",
    "                    metadata = self.metadata_func(item, {})\n",
    "                processed_data.append({'content': content, 'metadata': metadata})\n",
    "        return processed_data\n",
    "\n",
    "    def load(self) -> List[Document]:\n",
    "        \"\"\"\n",
    "        Load and return documents from the JSON file.\n",
    "        \"\"\"\n",
    "        docs = []\n",
    "        with open(self.file_path, mode=\"r\", encoding=\"utf-8\") as json_file:\n",
    "            try:\n",
    "                data = json.load(json_file)\n",
    "                processed_json = self.process_json(data)\n",
    "                docs = self.create_documents(processed_json)\n",
    "            except json.JSONDecodeError:\n",
    "                print(\"Error: Invalid JSON format in the file.\")\n",
    "        return docs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2179d868",
   "metadata": {},
   "source": [
    "- we will use the combined description for the semantic search\n",
    "- we will use price, # rooms, house size and neighborhood in the metadata for easier, more accurate 'keyword' search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0a064b87",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='Modern Luxury in Central Park: Indulge in the epitome of modern luxury with this stunning 4-bedroom, 3-bathroom home in Central Park. Boasting high-end finishes, a gourmet kitchen, and panoramic views of the city skyline, this property offers a sophisticated urban lifestyle. Neighborhood: Central Park is a vibrant neighborhood known for its upscale dining, shopping, and cultural attractions. Enjoy easy access to the park for leisurely strolls and outdoor activities.', metadata={'neighborhood': 'Central Park', 'price': 3500000, 'bedrooms': 4, 'bathrooms': 3, 'house size': 280}),\n",
       " Document(page_content='Seaside Retreat in Ocean View: Escape to this charming seaside retreat in Ocean View. With 3 bedrooms, 2 bathrooms, and a spacious deck overlooking the ocean, this home offers a perfect blend of coastal living and modern comfort. Neighborhood: Ocean View is a picturesque coastal community with sandy beaches, seafood restaurants, and a laid-back atmosphere. Experience the tranquility of seaside living in this charming neighborhood.', metadata={'neighborhood': 'Ocean View', 'price': 1800000, 'bedrooms': 3, 'bathrooms': 2, 'house size': 200}),\n",
       " Document(page_content='Historic Elegance in Heritage District: Step back in time with this historic 5-bedroom, 4-bathroom home in the Heritage District. Featuring elegant architecture, a grand staircase, and a beautifully landscaped garden, this property exudes timeless charm. Neighborhood: The Heritage District is a historic neighborhood filled with tree-lined streets, heritage homes, and boutique shops. Immerse yourself in the rich history and architectural beauty of this unique district.', metadata={'neighborhood': 'Heritage District', 'price': 1200000, 'bedrooms': 5, 'bathrooms': 4, 'house size': 320}),\n",
       " Document(page_content='Contemporary Living in Skyline Heights: Experience contemporary living at its finest in this 6-bedroom, 5-bathroom home in Skyline Heights. With sleek design, high ceilings, and a rooftop terrace with city views, this property is perfect for modern urban dwellers. Neighborhood: Skyline Heights offers a vibrant urban lifestyle with trendy cafes, art galleries, and easy access to public transportation. Enjoy the bustling energy of the city from the comfort of your stylish home.', metadata={'neighborhood': 'Skyline Heights', 'price': 2800000, 'bedrooms': 6, 'bathrooms': 5, 'house size': 400}),\n",
       " Document(page_content='Tranquil Haven in Willow Creek: Find peace and serenity in this tranquil 4-bedroom, 3-bathroom home in Willow Creek. Surrounded by lush greenery, with a private backyard and a cozy fireplace, this property offers a perfect retreat from the hustle and bustle of city life. Neighborhood: Willow Creek is a quiet residential neighborhood known for its tree-lined streets, parks, and family-friendly atmosphere. Enjoy a sense of community and nature in this peaceful enclave.', metadata={'neighborhood': 'Willow Creek', 'price': 1500000, 'bedrooms': 4, 'bathrooms': 3, 'house size': 250}),\n",
       " Document(page_content='Urban Chic in Downtown District: Embrace urban chic living in this stylish 3-bedroom, 2-bathroom home in the heart of the Downtown District. With modern design, high-end appliances, and a rooftop patio for entertaining, this property is perfect for those who love city living. Neighborhood: The Downtown District is a bustling urban center with a vibrant nightlife, trendy restaurants, and cultural attractions. Experience the excitement of city life right outside your doorstep.', metadata={'neighborhood': 'Downtown District', 'price': 2000000, 'bedrooms': 3, 'bathrooms': 2, 'house size': 180}),\n",
       " Document(page_content='Rustic Charm in Maple Grove: Discover the rustic charm of this 4-bedroom, 3-bathroom home in Maple Grove. With exposed wooden beams, a cozy fireplace, and a spacious backyard with a fire pit, this property offers a warm and inviting atmosphere. Neighborhood: Maple Grove is a quaint neighborhood with tree-lined streets, local cafes, and boutique shops. Enjoy a sense of community and small-town charm in this cozy enclave.', metadata={'neighborhood': 'Maple Grove', 'price': 1300000, 'bedrooms': 4, 'bathrooms': 3, 'house size': 220}),\n",
       " Document(page_content='Luxurious Retreat in Hillside Manor: Escape to a luxurious retreat in the prestigious Hillside Manor. This 5-bedroom, 4-bathroom home features a gourmet kitchen, a home theater, and a landscaped garden with a pool, offering the ultimate in upscale living. Neighborhood: Hillside Manor is an exclusive neighborhood known for its luxury estates, private golf courses, and scenic views. Enjoy privacy and sophistication in this upscale enclave.', metadata={'neighborhood': 'Hillside Manor', 'price': 3000000, 'bedrooms': 5, 'bathrooms': 4, 'house size': 350}),\n",
       " Document(page_content='Coastal Living in Seaview Heights: Experience the beauty of coastal living in this 3-bedroom, 2-bathroom home in Seaview Heights. With ocean views, a sun-drenched deck, and beach access just steps away, this property offers a perfect seaside retreat. Neighborhood: Seaview Heights is a coastal community with sandy beaches, seaside cafes, and water sports activities. Enjoy the laid-back lifestyle and stunning ocean views in this charming neighborhood.', metadata={'neighborhood': 'Seaview Heights', 'price': 2200000, 'bedrooms': 3, 'bathrooms': 2, 'house size': 190}),\n",
       " Document(page_content='Elegant Oasis in Rosewood Estates: Discover an elegant oasis in the prestigious Rosewood Estates. This 4-bedroom, 3-bathroom home features a grand foyer, a gourmet kitchen, and a landscaped backyard with a fountain, offering a perfect blend of luxury and sophistication. Neighborhood: Rosewood Estates is an upscale neighborhood known for its luxury homes, private estates, and exclusive country clubs. Experience the epitome of luxury living in this prestigious enclave.', metadata={'neighborhood': 'Rosewood Estates', 'price': 2600000, 'bedrooms': 4, 'bathrooms': 3, 'house size': 240}),\n",
       " Document(page_content=\"Serenity in Pinecrest Meadows: Find serenity in this 5-bedroom, 4-bathroom home in Pinecrest Meadows. Surrounded by towering pine trees, with a wrap-around porch and a cozy fireplace, this property offers a peaceful retreat in a natural setting. Neighborhood: Pinecrest Meadows is a nature lover's paradise with hiking trails, wildlife, and scenic views. Enjoy the tranquility and beauty of the outdoors in this idyllic neighborhood.\", metadata={'neighborhood': 'Pinecrest Meadows', 'price': 1700000, 'bedrooms': 5, 'bathrooms': 4, 'house size': 260}),\n",
       " Document(page_content='Modern Elegance in Silver Lake: Experience modern elegance in this 3-bedroom, 2-bathroom home in Silver Lake. With sleek design, high ceilings, and a rooftop deck with panoramic views of the lake, this property offers a perfect blend of style and sophistication. Neighborhood: Silver Lake is a trendy neighborhood known for its hip cafes, art galleries, and scenic lake views. Embrace the vibrant energy and creative spirit of this eclectic community.', metadata={'neighborhood': 'Silver Lake', 'price': 1900000, 'bedrooms': 3, 'bathrooms': 2, 'house size': 210}),\n",
       " Document(page_content='Charming Cottage in Willowbrook Village: Escape to this charming cottage in the heart of Willowbrook Village. With 2 bedrooms, 1 bathroom, a cozy fireplace, and a flower-filled garden, this property offers a perfect retreat for those seeking a peaceful lifestyle. Neighborhood: Willowbrook Village is a quaint village with historic cottages, local bakeries, and community events. Enjoy a sense of nostalgia and small-town charm in this idyllic setting.', metadata={'neighborhood': 'Willowbrook Village', 'price': 1400000, 'bedrooms': 2, 'bathrooms': 1, 'house size': 150}),\n",
       " Document(page_content='Luxury Living in Diamond Heights: Indulge in luxury living in this 4-bedroom, 3-bathroom home in Diamond Heights. With high-end finishes, a gourmet kitchen, and a landscaped backyard with a pool, this property offers the ultimate in upscale living. Neighborhood: Diamond Heights is an exclusive neighborhood known for its luxury estates, private tennis clubs, and panoramic views. Experience privacy and sophistication in this prestigious enclave.', metadata={'neighborhood': 'Diamond Heights', 'price': 2700000, 'bedrooms': 4, 'bathrooms': 3, 'house size': 230}),\n",
       " Document(page_content='Tranquil Retreat in Cedar Valley: Find peace and tranquility in this 3-bedroom, 2-bathroom home in Cedar Valley. Surrounded by cedar trees, with a wrap-around porch and a meditation garden, this property offers a serene retreat in a natural setting. Neighborhood: Cedar Valley is a peaceful neighborhood with hiking trails, nature reserves, and a strong sense of community. Enjoy the beauty of the outdoors and a slower pace of life in this idyllic enclave.', metadata={'neighborhood': 'Cedar Valley', 'price': 1600000, 'bedrooms': 3, 'bathrooms': 2, 'house size': 170}),\n",
       " Document(page_content='Urban Oasis in City Center: Discover an urban oasis in the heart of the City Center. This 4-bedroom, 3-bathroom home features modern design, a rooftop terrace with city views, and a home office, offering a perfect blend of style and functionality. Neighborhood: City Center is a vibrant urban hub with shopping centers, restaurants, and cultural attractions. Experience the convenience and excitement of city living in this bustling neighborhood.', metadata={'neighborhood': 'City Center', 'price': 2500000, 'bedrooms': 4, 'bathrooms': 3, 'house size': 260})]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def metadata_func(item, _):\n",
    "    return {\n",
    "            'neighborhood':item['listings_neighborhood'],\n",
    "            'price': item['listings_price'],\n",
    "            'bedrooms':item['listings_bedrooms'],\n",
    "            'bathrooms':item['listings_bathrooms'],\n",
    "            'house size':item['listings_house_size'],\n",
    "            }\n",
    "\n",
    "json_loader= JSONLoader('data/property_listings.json',content_key='full_description', metadata_func=metadata_func)\n",
    "docs = json_loader.load()\n",
    "docs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae8aea72",
   "metadata": {},
   "source": [
    "- For demonstration purpose we implement a text splitter. However to ensure semantic search we embed the full description as a whole since it is of limited size anyhow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "96dab035",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''# Splitter to chunk text into smaller chunks\n",
    "text_splitter = CharacterTextSplitter(\n",
    "    separator=\"\\n\\n\",\n",
    "    chunk_size=200,\n",
    "    chunk_overlap=20,\n",
    "    length_function=len,\n",
    "    is_separator_regex=False,\n",
    ")\n",
    "\n",
    "split_docs = text_splitter.split_documents(docs)\n",
    "\n",
    "# embed each chunk and load it into the vector store.\n",
    "db = Chroma.from_documents(split_docs, OpenAIEmbeddings())'''\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fe61496",
   "metadata": {},
   "source": [
    "### RUBRIC #2 Creating a Vector Database and Storing Listings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "66815237",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we create a chromadb collection to store the listings, their embeddings and metadata\n",
    "try:\n",
    "    collection = client.get_collection(name=\"homematch\", embedding_function=OpenAIEmbeddings())\n",
    "except:\n",
    "    collection = client.create_collection(name=\"homematch\", embedding_function=OpenAIEmbeddings())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f5bf0cc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we generate the ambeddings for each description using OpenAi Embedding model\n",
    "embeddings = embeddings_model.embed_documents([doc.page_content for doc in docs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "6a400873",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we add the data to the collection - this is automatically persisted to disk in the data/db folder\n",
    "collection.add(\n",
    "    documents=[doc.page_content for doc in docs],\n",
    "    embeddings=embeddings,\n",
    "    metadatas=[doc.metadata for doc in docs],\n",
    "    ids=[str(i) for i in range(len(docs))] \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "0343d542",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "collection.count() # returns the number of items in the collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "c8ab3eba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ids': ['0', '1'],\n",
       " 'embeddings': [[0.012835898850191383,\n",
       "   0.023745753775894586,\n",
       "   -0.009729163328777135,\n",
       "   -0.013508695564182034,\n",
       "   0.0022014292466056174,\n",
       "   0.018007195759674403,\n",
       "   -0.014379373171587419,\n",
       "   0.0029847092188278394,\n",
       "   -0.03350789709835375,\n",
       "   -0.0130139919336614,\n",
       "   -0.0005907580833086552,\n",
       "   0.01776973831504771,\n",
       "   -0.022255047258125295,\n",
       "   -0.005339496553641952,\n",
       "   -0.0010479462375994788,\n",
       "   0.0002786828150790133,\n",
       "   0.0322414549101513,\n",
       "   -0.017505896502946375,\n",
       "   0.011490306353532613,\n",
       "   -0.02121287312477982,\n",
       "   -0.036805918818297874,\n",
       "   0.0114045576249013,\n",
       "   0.006371777247676338,\n",
       "   -0.011648611627057921,\n",
       "   -0.004933839930517597,\n",
       "   -0.0013662053419559895,\n",
       "   0.010843894007016602,\n",
       "   -0.008066960285067752,\n",
       "   0.011364982005011875,\n",
       "   0.0029962524960133116,\n",
       "   0.004854687293754943,\n",
       "   -0.0063289028833606814,\n",
       "   -0.027360386042889324,\n",
       "   -0.005398860914798625,\n",
       "   -0.00310014012803749,\n",
       "   -0.017756547062632928,\n",
       "   -0.004171997140453203,\n",
       "   -0.020988608795212985,\n",
       "   0.0076316214813650605,\n",
       "   -0.0032666901671526707,\n",
       "   0.013996802637172739,\n",
       "   -0.025289227165968084,\n",
       "   0.006787327775773049,\n",
       "   -0.0012928243437860722,\n",
       "   -0.009828103309823233,\n",
       "   -3.751499237864789e-05,\n",
       "   -6.462060731259697e-05,\n",
       "   -0.027967220906870836,\n",
       "   -0.0445364807463703,\n",
       "   -0.015975616227932737,\n",
       "   0.014352988804112777,\n",
       "   0.022334200360549217,\n",
       "   -0.023099341429378573,\n",
       "   -0.010672397481076512,\n",
       "   -0.01986728155944359,\n",
       "   -0.01240056051214996,\n",
       "   -0.02717569547056684,\n",
       "   0.0009712673099105484,\n",
       "   0.00999960076708586,\n",
       "   -0.02707015986331335,\n",
       "   -0.002958325084183833,\n",
       "   -0.023521487583682676,\n",
       "   -0.015883270010448956,\n",
       "   -0.002470218011193128,\n",
       "   -0.007552468844602408,\n",
       "   -0.008475915186957072,\n",
       "   -0.04543354178992779,\n",
       "   0.004185189324190523,\n",
       "   0.006906056498086396,\n",
       "   -0.0009325155612204628,\n",
       "   0.040103940538242,\n",
       "   0.007684389750653073,\n",
       "   0.0009564262196213801,\n",
       "   0.004139016681109902,\n",
       "   0.010995603654334518,\n",
       "   -0.017545473985480876,\n",
       "   -0.005890266266554294,\n",
       "   0.026872278038576083,\n",
       "   -0.0015418249491829196,\n",
       "   0.007783330197360439,\n",
       "   0.005042674747858586,\n",
       "   -0.03329682215855663,\n",
       "   -0.0061508097998906625,\n",
       "   0.025302420281027943,\n",
       "   0.02658205185900011,\n",
       "   0.027334001675414683,\n",
       "   -0.02546072462323071,\n",
       "   0.013970418269698099,\n",
       "   0.008541875174321137,\n",
       "   0.004152208864847223,\n",
       "   -0.012756746679089997,\n",
       "   -0.011345193263744627,\n",
       "   0.0011172047365591421,\n",
       "   0.007710774118127714,\n",
       "   -0.039549870683919616,\n",
       "   0.023587449433691815,\n",
       "   -0.0009902308994099707,\n",
       "   0.017439936515582313,\n",
       "   -0.008132920272431817,\n",
       "   0.017796122682522352,\n",
       "   0.009062962240993873,\n",
       "   0.003522286748002861,\n",
       "   -0.015777734403195468,\n",
       "   -0.011444134176113261,\n",
       "   -0.04187168012052741,\n",
       "   -0.010171098224348486,\n",
       "   0.01907575612313959,\n",
       "   -0.026344594414373416,\n",
       "   0.035697782834943265,\n",
       "   0.045407159285098225,\n",
       "   -0.017189287818540837,\n",
       "   0.01160903507584596,\n",
       "   0.024721967921875997,\n",
       "   -0.036489306408602186,\n",
       "   0.00700499741045503,\n",
       "   -0.0011510094163477327,\n",
       "   0.0027670398169764936,\n",
       "   -0.030711174635137655,\n",
       "   -0.011516690721007255,\n",
       "   0.010975814913067268,\n",
       "   0.008357186464643725,\n",
       "   0.015012593334366109,\n",
       "   0.027782532197193427,\n",
       "   0.010309614756606545,\n",
       "   -0.01239396395462003,\n",
       "   -0.0030457227193669946,\n",
       "   -0.004287427584001585,\n",
       "   -0.02944473430958027,\n",
       "   -0.04643614030338384,\n",
       "   0.003984009686349559,\n",
       "   -0.004854687293754943,\n",
       "   0.0025988408713094645,\n",
       "   0.024748352289350635,\n",
       "   -0.0034926045674245247,\n",
       "   -0.014735559338527457,\n",
       "   0.0100259851345605,\n",
       "   -0.0020431242059109444,\n",
       "   0.012994204123716688,\n",
       "   -0.0320831524305936,\n",
       "   -0.032821907269303244,\n",
       "   0.028521288898548144,\n",
       "   0.032452529849948425,\n",
       "   -0.011325405453799915,\n",
       "   0.030948632079764348,\n",
       "   0.013469119012970073,\n",
       "   0.005276834379381583,\n",
       "   0.0006888741931554139,\n",
       "   0.02923366123242822,\n",
       "   0.008700180447846444,\n",
       "   -0.008594643909270418,\n",
       "   0.00822526555859306,\n",
       "   -0.03664761261345003,\n",
       "   0.012690785760403394,\n",
       "   0.015724965668246185,\n",
       "   -0.012888667585140663,\n",
       "   0.018983411768300887,\n",
       "   -0.0008748001648232962,\n",
       "   0.01609434495024608,\n",
       "   -0.014933440231942187,\n",
       "   -0.0015748051756955862,\n",
       "   0.004247851498450892,\n",
       "   0.0042181693178725554,\n",
       "   -0.004373176312632898,\n",
       "   -0.0014272187260798276,\n",
       "   -0.009452129332938482,\n",
       "   0.012849091033928702,\n",
       "   0.017901660152420912,\n",
       "   0.028574057633497425,\n",
       "   -0.024748352289350635,\n",
       "   0.009729163328777135,\n",
       "   -0.004973416016068289,\n",
       "   0.025975216995018594,\n",
       "   -0.03147631756661209,\n",
       "   0.016147111822550287,\n",
       "   0.006952229141167017,\n",
       "   0.01776973831504771,\n",
       "   0.016965021626328926,\n",
       "   -0.007222666579475742,\n",
       "   -0.0035981413388311845,\n",
       "   -0.012360983960937998,\n",
       "   -0.0022459525174731224,\n",
       "   -0.006084849346865329,\n",
       "   0.0015113183153286592,\n",
       "   0.044879475660895556,\n",
       "   0.0016308716073332463,\n",
       "   0.004244553685347196,\n",
       "   0.035618631595164416,\n",
       "   -0.02939196557463099,\n",
       "   -0.004696382485890905,\n",
       "   -0.01419468353058747,\n",
       "   0.009735758954984526,\n",
       "   0.0018666800289927734,\n",
       "   0.023534680698742535,\n",
       "   -0.021476714936881153,\n",
       "   -0.6425073483710605,\n",
       "   -0.0009399361063650469,\n",
       "   -0.02083030259036514,\n",
       "   -0.020922646945203845,\n",
       "   0.030922247712289706,\n",
       "   0.030869478977340426,\n",
       "   0.014761943706002097,\n",
       "   0.02308615017696379,\n",
       "   -0.0004312163044924377,\n",
       "   -0.025671797700382765,\n",
       "   -0.0009152009364805469,\n",
       "   -0.005039376469093622,\n",
       "   -0.0003238244841111196,\n",
       "   0.004821707300072911,\n",
       "   -0.026357787529433272,\n",
       "   -0.010283230389131905,\n",
       "   0.0023234560148532937,\n",
       "   -0.013746153008808727,\n",
       "   -0.03171377501123878,\n",
       "   -0.006434439887597975,\n",
       "   0.014564061881264828,\n",
       "   0.02208355166350774,\n",
       "   -0.010296422572869224,\n",
       "   -0.016213073672559426,\n",
       "   -0.0046337198459692665,\n",
       "   0.005055866931595907,\n",
       "   0.019629822252171823,\n",
       "   -0.011299021086325274,\n",
       "   -0.012512692676933376,\n",
       "   -0.0017759844642905988,\n",
       "   -0.01201139342020535,\n",
       "   0.02730761730794004,\n",
       "   -0.0017199181490682558,\n",
       "   0.008376974274588438,\n",
       "   0.04572376796950376,\n",
       "   -0.0248143122767147,\n",
       "   -0.03018349101093499,\n",
       "   0.024906656631553406,\n",
       "   -0.004587547668549914,\n",
       "   0.030684790267663013,\n",
       "   -0.005032780377224962,\n",
       "   0.012908455395085375,\n",
       "   0.01402318700464738,\n",
       "   0.001444533409027402,\n",
       "   0.005075654741540619,\n",
       "   0.018943834285766386,\n",
       "   0.02743953728266817,\n",
       "   -0.010725165284703256,\n",
       "   0.004221467130976251,\n",
       "   0.001073505918967561,\n",
       "   0.022716770894963895,\n",
       "   -0.007117130040899715,\n",
       "   -0.015738158783306044,\n",
       "   0.017994004507259617,\n",
       "   0.004092844503690549,\n",
       "   -0.005408754819770981,\n",
       "   0.023297223254115842,\n",
       "   -0.010204078218030519,\n",
       "   0.018877874298402323,\n",
       "   -0.009076154424731194,\n",
       "   0.01817869321693703,\n",
       "   0.005527484007745595,\n",
       "   -0.014656407167426071,\n",
       "   -0.022123127283397166,\n",
       "   -0.012783131046564637,\n",
       "   0.03427303816718311,\n",
       "   -0.003792724651972854,\n",
       "   0.0023597342873002904,\n",
       "   -0.012545673601937946,\n",
       "   -0.022096742915922524,\n",
       "   -0.006655406869722383,\n",
       "   0.0243921661224106,\n",
       "   -0.014511294077638084,\n",
       "   -0.013917650466071354,\n",
       "   0.019339597935240923,\n",
       "   -0.0049305421174139,\n",
       "   0.032848293499422955,\n",
       "   -0.0191285229954438,\n",
       "   -0.013521887747919355,\n",
       "   0.008700180447846444,\n",
       "   0.013838497363647432,\n",
       "   -0.015883270010448956,\n",
       "   -0.022162702903286587,\n",
       "   -0.022479313450337202,\n",
       "   0.010448131288864604,\n",
       "   -0.014973016783154148,\n",
       "   -0.026674396213838814,\n",
       "   0.010085349495717173,\n",
       "   0.026555667491525468,\n",
       "   -0.014709174971052815,\n",
       "   0.013277833745762734,\n",
       "   0.0010355787399687166,\n",
       "   -0.010758145278385289,\n",
       "   -0.02224185600571051,\n",
       "   -0.03461603308170837,\n",
       "   0.018877874298402323,\n",
       "   0.004475415038105228,\n",
       "   0.02820467835149753,\n",
       "   -0.00701818959419235,\n",
       "   -0.02704377549583871,\n",
       "   -0.01185308907800258,\n",
       "   -0.0075062966671830544,\n",
       "   0.02780891656466807,\n",
       "   -0.005415350911639641,\n",
       "   0.02658205185900011,\n",
       "   0.0031133320789441758,\n",
       "   0.0007498875772792519,\n",
       "   0.030737559002612293,\n",
       "   0.03137078009671353,\n",
       "   -0.022479313450337202,\n",
       "   -0.02725484857299076,\n",
       "   -0.012835898850191383,\n",
       "   -0.016516491104550186,\n",
       "   -0.004300619767738905,\n",
       "   0.025302420281027943,\n",
       "   -0.021569059291719858,\n",
       "   -0.001965620708530773,\n",
       "   0.008647411712897162,\n",
       "   0.019115331743029016,\n",
       "   -0.00997321639961122,\n",
       "   0.001947481572307275,\n",
       "   -0.006401459428254675,\n",
       "   0.013680193021444662,\n",
       "   -0.014735559338527457,\n",
       "   0.009254247508201214,\n",
       "   0.002300369926143617,\n",
       "   0.023877673750622715,\n",
       "   -0.02367979378853052,\n",
       "   0.0017397063082589192,\n",
       "   0.0009506546392363026,\n",
       "   0.012862283217666023,\n",
       "   0.012209275244942619,\n",
       "   0.0382570459908876,\n",
       "   -0.0019013092784726051,\n",
       "   0.008436338635745111,\n",
       "   0.020632420765627872,\n",
       "   0.010962622729329949,\n",
       "   -0.02579052642269611,\n",
       "   0.014471717526426123,\n",
       "   -0.02208355166350774,\n",
       "   -0.03509094797096175,\n",
       "   0.010263442579187192,\n",
       "   -0.0037234661530131906,\n",
       "   0.0038125126947482005,\n",
       "   -0.00918828752083715,\n",
       "   -0.0006868129435502868,\n",
       "   -0.017954428887370196,\n",
       "   -0.009867679861035194,\n",
       "   -0.04651529154316269,\n",
       "   -0.009300420616943103,\n",
       "   -0.048837100979770474,\n",
       "   -0.015157706424154095,\n",
       "   -0.02949750304452955,\n",
       "   0.01645053111718612,\n",
       "   0.011556267272219216,\n",
       "   -0.005329602648669596,\n",
       "   0.01755866523789566,\n",
       "   -0.018732761208614335,\n",
       "   -0.012756746679089997,\n",
       "   -0.0104217478527125,\n",
       "   0.015395163868780789,\n",
       "   0.0006192035257654472,\n",
       "   -0.026120328222161506,\n",
       "   0.01722886343843026,\n",
       "   0.017730162695158286,\n",
       "   -0.010507495650021277,\n",
       "   0.009161903153362507,\n",
       "   0.0012581950943062405,\n",
       "   -0.004307215859607565,\n",
       "   -0.045486310524877074,\n",
       "   -0.0028165102731608107,\n",
       "   -0.014326604436638137,\n",
       "   -0.0132448537520807,\n",
       "   0.017255247805904903,\n",
       "   0.017994004507259617,\n",
       "   -0.0004378123381534394,\n",
       "   -0.0017116730342324306,\n",
       "   -0.03754467365700752,\n",
       "   -0.022505697817811844,\n",
       "   0.006124425432416022,\n",
       "   0.018165501964522247,\n",
       "   -0.0027521987266873253,\n",
       "   -0.037861286066703206,\n",
       "   -0.008964022259947777,\n",
       "   0.03234699238004986,\n",
       "   0.007433740122289061,\n",
       "   0.036515692638721904,\n",
       "   0.01699140599380357,\n",
       "   -0.004472116759340264,\n",
       "   0.011371577631219269,\n",
       "   0.028758746343174837,\n",
       "   0.020500500790899743,\n",
       "   -0.02191205420624511,\n",
       "   0.009287228433205784,\n",
       "   0.009122326602150546,\n",
       "   0.010665800923546583,\n",
       "   0.014735559338527457,\n",
       "   0.0004485309001285243,\n",
       "   0.0006629022851493693,\n",
       "   -0.005329602648669596,\n",
       "   0.0024372377846804618,\n",
       "   0.011905856881629324,\n",
       "   0.00781631065670374,\n",
       "   -0.026595244974059965,\n",
       "   0.027888067804446914,\n",
       "   -0.04102738408662905,\n",
       "   0.009894064228509834,\n",
       "   0.007150110500243017,\n",
       "   0.010355786934025897,\n",
       "   0.007552468844602408,\n",
       "   0.004135718868006206,\n",
       "   -0.005395563101694929,\n",
       "   -0.020474116423425105,\n",
       "   -0.008726564815321084,\n",
       "   0.0315818513112205,\n",
       "   0.029682191754206964,\n",
       "   0.015118129872942134,\n",
       "   0.025566262093129274,\n",
       "   0.00803398029138572,\n",
       "   -0.00939276497178181,\n",
       "   0.0325052985848977,\n",
       "   0.005306516559959919,\n",
       "   0.007486508391577074,\n",
       "   0.006312412886519665,\n",
       "   -0.04693744142275694,\n",
       "   0.011899261255421933,\n",
       "   -0.013099740662292714,\n",
       "   0.021410754949517087,\n",
       "   -0.012677594507988612,\n",
       "   -0.02730761730794004,\n",
       "   -0.02637097878184806,\n",
       "   0.035249250450519444,\n",
       "   0.021964822941194395,\n",
       "   0.00678073168390439,\n",
       "   -0.02023665897879841,\n",
       "   0.019933241546807656,\n",
       "   0.006549870331146357,\n",
       "   -0.024761543541765418,\n",
       "   0.017176094703480978,\n",
       "   -0.007070957863480363,\n",
       "   0.01820507758441167,\n",
       "   0.014959824599416829,\n",
       "   -0.0008286278709886263,\n",
       "   -0.019603439747342258,\n",
       "   -0.007499700575314395,\n",
       "   0.000472853814271233,\n",
       "   0.027914452171921556,\n",
       "   0.029893264831359016,\n",
       "   -0.004821707300072911,\n",
       "   0.004923946025545241,\n",
       "   -0.016503297989490327,\n",
       "   -0.01609434495024608,\n",
       "   -0.0032155708044165057,\n",
       "   0.012822706666454062,\n",
       "   0.004092844503690549,\n",
       "   -0.008317609913431764,\n",
       "   0.008139516829961746,\n",
       "   -0.013106337219822644,\n",
       "   0.012281831789836613,\n",
       "   0.03530201918546873,\n",
       "   0.0014412353630930718,\n",
       "   -0.025750950802806687,\n",
       "   0.019115331743029016,\n",
       "   -0.00918828752083715,\n",
       "   0.013469119012970073,\n",
       "   -0.0024372377846804618,\n",
       "   0.01617349619002493,\n",
       "   -0.042346595009780795,\n",
       "   0.01981451282449431,\n",
       "   0.00037247028329270767,\n",
       "   0.02077753385541586,\n",
       "   -0.028679593240750912,\n",
       "   0.032821907269303244,\n",
       "   -0.015883270010448956,\n",
       "   -0.020197083358908987,\n",
       "   -0.014748751522264776,\n",
       "   -0.0002743541443421197,\n",
       "   -0.012987608497509297,\n",
       "   -0.018495303763987645,\n",
       "   0.01652968235696497,\n",
       "   0.014471717526426123,\n",
       "   -0.010685588733491295,\n",
       "   0.013864881731122073,\n",
       "   0.012130122142518697,\n",
       "   0.0065069959668307,\n",
       "   -0.008330802097169085,\n",
       "   -0.010058965128242532,\n",
       "   0.01882510556345304,\n",
       "   0.02370617815600516,\n",
       "   0.04730681884211176,\n",
       "   0.02002558590164636,\n",
       "   0.010520687833758596,\n",
       "   -0.006216770252915995,\n",
       "   0.005741855363662611,\n",
       "   -0.002447131922483452,\n",
       "   -0.01817869321693703,\n",
       "   0.05144913287066409,\n",
       "   -0.012314811783518644,\n",
       "   0.0009819859009894626,\n",
       "   -0.03329682215855663,\n",
       "   -0.015250050778992802,\n",
       "   0.011312213270062595,\n",
       "   0.005385668731061304,\n",
       "   -0.00336233280075634,\n",
       "   -0.007651409756971041,\n",
       "   0.010547072201233238,\n",
       "   0.0037663402844982136,\n",
       "   0.0030984909886550074,\n",
       "   0.001070207873033231,\n",
       "   -0.028547673266022783,\n",
       "   0.00047656408684352506,\n",
       "   -0.01864041685377563,\n",
       "   -0.009854487677297873,\n",
       "   -0.011134120186592576,\n",
       "   -0.01647691548466076,\n",
       "   -0.009544473687777187,\n",
       "   -0.009471917142883193,\n",
       "   0.02443174174230002,\n",
       "   0.010329402566551257,\n",
       "   -0.030368179720612398,\n",
       "   -0.005534079633952987,\n",
       "   -0.03026264411335891,\n",
       "   -0.017782931430107566,\n",
       "   0.027334001675414683,\n",
       "   0.10817510198335484,\n",
       "   -0.013112932846030035,\n",
       "   0.005725364901160326,\n",
       "   0.03757105988712724,\n",
       "   0.0016828154233453352,\n",
       "   -0.006309115073415969,\n",
       "   -0.015184089860306199,\n",
       "   0.0006563062514883677,\n",
       "   0.013838497363647432,\n",
       "   0.011325405453799915,\n",
       "   0.005764941452372287,\n",
       "   -0.002288826881788779,\n",
       "   0.00042173448063889733,\n",
       "   0.005764941452372287,\n",
       "   0.009471917142883193,\n",
       "   -0.0004885194160439837,\n",
       "   -0.01994643279922244,\n",
       "   -0.04680551772273866,\n",
       "   -0.040024785573173,\n",
       "   -0.00246527105870695,\n",
       "   0.01980131970943445,\n",
       "   -0.003202378620679185,\n",
       "   0.003032530535629674,\n",
       "   0.030289028480833553,\n",
       "   -0.004125824497372582,\n",
       "   0.01400999482091006,\n",
       "   0.016054767467711582,\n",
       "   -0.007103937857162395,\n",
       "   0.017598242720430157,\n",
       "   -0.031106938284612192,\n",
       "   0.007849290650385773,\n",
       "   0.0007119603400727487,\n",
       "   -0.006833500418853671,\n",
       "   0.00025889468499217915,\n",
       "   -0.020276234598687836,\n",
       "   -0.0014206227506264844,\n",
       "   0.0017611433740014305,\n",
       "   -0.02240016034791328,\n",
       "   0.032109534935423165,\n",
       "   -0.0023580853807484425,\n",
       "   0.02579052642269611,\n",
       "   0.009307016243150494,\n",
       "   0.012380771770882709,\n",
       "   -0.004066460136215908,\n",
       "   0.010428343478919891,\n",
       "   0.0005643738904569901,\n",
       "   -0.010342594750288578,\n",
       "   0.03617269772419665,\n",
       "   0.006243154620390636,\n",
       "   0.011985009984053246,\n",
       "   0.006678493424093328,\n",
       "   0.004221467130976251,\n",
       "   -0.0008001824285318343,\n",
       "   -0.0005087198018726091,\n",
       "   0.03308575094404965,\n",
       "   -0.007935039379017086,\n",
       "   0.024009595587995917,\n",
       "   -0.02002558590164636,\n",
       "   -0.007981211556436439,\n",
       "   -0.030948632079764348,\n",
       "   0.00521087392635625,\n",
       "   -0.04348111349796497,\n",
       "   0.003403558025689515,\n",
       "   -0.007532680568996427,\n",
       "   -0.016687988561812812,\n",
       "   0.0021618529282242905,\n",
       "   -0.001016615034053977,\n",
       "   0.01747951399811681,\n",
       "   -0.03213592116554288,\n",
       "   -0.011820108152998011,\n",
       "   -0.0004062749921851275,\n",
       "   -0.0024883571474166267,\n",
       "   -0.0020348792074904366,\n",
       "   -0.010962622729329949,\n",
       "   0.03429942067201268,\n",
       "   -0.003792724651972854,\n",
       "   0.024128324310309264,\n",
       "   0.004943733835489952,\n",
       "   -0.009161903153362507,\n",
       "   0.03472157055160693,\n",
       "   0.007697581934390394,\n",
       "   -0.024101939942834626,\n",
       "   0.012888667585140663,\n",
       "   -0.007249050946950382,\n",
       "   0.00037741732309037353,\n",
       "   0.004244553685347196,\n",
       "   0.004689786394022244,\n",
       "   -0.0188382986785129,\n",
       "   -0.001223565961241726,\n",
       "   0.014405757539062059,\n",
       "   0.009102538792205834,\n",
       "   0.0019837598447542716,\n",
       "   0.01120667673148657,\n",
       "   0.03152908257627122,\n",
       "   -0.006375075060780034,\n",
       "   -0.017387167780633032,\n",
       "   -0.009069558798523803,\n",
       "   0.0036377176572125114,\n",
       "   0.017492705250531593,\n",
       "   -0.006840096510722331,\n",
       "   0.011978413426523319,\n",
       "   -0.010909854925703203,\n",
       "   0.004412752398183591,\n",
       "   -0.0046469120297065875,\n",
       "   0.0104217478527125,\n",
       "   -0.0027225165461089887,\n",
       "   -0.016371378014762197,\n",
       "   -0.0047359585714415965,\n",
       "   0.0002071569677471566,\n",
       "   0.023521487583682676,\n",
       "   0.015527084774831454,\n",
       "   -0.020764342603001078,\n",
       "   -0.010698780917228616,\n",
       "   0.014366180987850098,\n",
       "   -0.017519089618006235,\n",
       "   0.004096142316794245,\n",
       "   0.00012212985401099594,\n",
       "   0.02833660018887073,\n",
       "   0.010560264384970557,\n",
       "   -0.01598880748034752,\n",
       "   -0.015540276958568775,\n",
       "   -0.02561902896543348,\n",
       "   0.011872876887947293,\n",
       "   0.01882510556345304,\n",
       "   0.017703778327683644,\n",
       "   -0.0016399411754449956,\n",
       "   0.005543974004586612,\n",
       "   0.023323607621590484,\n",
       "   -0.006345392880201697,\n",
       "   0.007216070487607081,\n",
       "   -0.006919248681823716,\n",
       "   0.020817111337950358,\n",
       "   -0.02036857895352654,\n",
       "   -0.04572376796950376,\n",
       "   -0.0060386771694459765,\n",
       "   -0.010118329489399206,\n",
       "   -0.006741155598353697,\n",
       "   0.005692384907478294,\n",
       "   -0.018548072498936925,\n",
       "   -0.00904977098857909,\n",
       "   -0.025566262093129274,\n",
       "   0.01181351252679062,\n",
       "   -0.003993904056983184,\n",
       "   -0.01181351252679062,\n",
       "   0.003627823519409521,\n",
       "   -0.021661403646558563,\n",
       "   0.009828103309823233,\n",
       "   -0.034061963227385986,\n",
       "   -0.022769539629913176,\n",
       "   0.031186089524391038,\n",
       "   0.0003854562372957299,\n",
       "   0.006121127619312326,\n",
       "   0.0033804719369798387,\n",
       "   0.020144314623959707,\n",
       "   -0.0215954436591945,\n",
       "   -0.01085049056454653,\n",
       "   0.0024949532392852867,\n",
       "   -0.015289627330204763,\n",
       "   0.007130322224637036,\n",
       "   0.020302618966162474,\n",
       "   0.004966820389860897,\n",
       "   -0.014102339175748765,\n",
       "   0.0033227564823750137,\n",
       "   0.0030885970836826513,\n",
       "   0.010256846021657263,\n",
       "   0.009735758954984526,\n",
       "   -0.018614032486300988,\n",
       "   -0.0007049520797737851,\n",
       "   -0.009702778961302495,\n",
       "   0.022967421454650444,\n",
       "   0.01842934377662358,\n",
       "   0.01742674526316753,\n",
       "   0.00997321639961122,\n",
       "   -0.014141915726960726,\n",
       "   -0.02272996401002375,\n",
       "   0.03187207749079647,\n",
       "   0.00039658705501260607,\n",
       "   -0.010032580760767892,\n",
       "   -0.012334599593463356,\n",
       "   -0.01980131970943445,\n",
       "   -0.027492306017617454,\n",
       "   0.020540076410789167,\n",
       "   -0.05693704032719772,\n",
       "   -0.0013653808886800655,\n",
       "   -0.008594643909270418,\n",
       "   -0.005174595653909253,\n",
       "   0.02833660018887073,\n",
       "   0.009485109326620514,\n",
       "   0.007163302218319068,\n",
       "   -0.0023646812397864684,\n",
       "   0.00037391316383706243,\n",
       "   -0.040367780487698256,\n",
       "   0.02321807015169192,\n",
       "   0.01864041685377563,\n",
       "   0.010124925115606597,\n",
       "   0.00965660678388314,\n",
       "   0.008924445708735816,\n",
       "   -0.04337557602806641,\n",
       "   0.005880371895920669,\n",
       "   0.028521288898548144,\n",
       "   -0.0007395812710459583,\n",
       "   0.004175294953556899,\n",
       "   -0.016674795446752953,\n",
       "   -0.01877233682850376,\n",
       "   0.02815190961654825,\n",
       "   0.028547673266022783,\n",
       "   -0.0096302224164085,\n",
       "   -0.009689586777565174,\n",
       "   0.004881071661229584,\n",
       "   0.004231361501609876,\n",
       "   0.003502498705227515,\n",
       "   -0.014247452265536751,\n",
       "   -0.011477114169795294,\n",
       "   -0.035776934074722114,\n",
       "   0.006173895888600339,\n",
       "   0.008363782090851118,\n",
       "   0.0023036679720779473,\n",
       "   0.020592845145738448,\n",
       "   -0.028099142744244042,\n",
       "   0.004834899483810231,\n",
       "   0.02118648875730518,\n",
       "   -0.006282730705941329,\n",
       "   0.011450729802320652,\n",
       "   -0.013732960825071406,\n",
       "   0.0046601042134439075,\n",
       "   -0.005306516559959919,\n",
       "   -0.0016514842197998337,\n",
       "   0.017901660152420912,\n",
       "   0.017308016540854183,\n",
       "   -0.009788527689933808,\n",
       "   0.0023069660180122776,\n",
       "   0.017334400908328825,\n",
       "   0.006305816794651005,\n",
       "   0.01565900568088212,\n",
       "   -0.00782950284044106,\n",
       "   -0.007776734571153048,\n",
       "   -0.0028775235408693314,\n",
       "   -0.0197221684696556,\n",
       "   -0.0066685990534597036,\n",
       "   0.0480455736808214,\n",
       "   0.008825504796367182,\n",
       "   0.004109334500531566,\n",
       "   -0.020038777154061143,\n",
       "   0.0015261594056178273,\n",
       "   0.002283879696471967,\n",
       "   0.02447131922483452,\n",
       "   0.0020579652962001128,\n",
       "   0.019431942290079628,\n",
       "   -0.0044688189462365685,\n",
       "   0.007862482834123092,\n",
       "   -0.0300779554036815,\n",
       "   0.034800721791385776,\n",
       "   -0.00679392386764171,\n",
       "   0.015395163868780789,\n",
       "   -0.011147312370329896,\n",
       "   -0.045565461764655917,\n",
       "   -0.0020991905211332875,\n",
       "   -0.0035585650204498576,\n",
       "   -0.015421548236255429,\n",
       "   0.014247452265536751,\n",
       "   0.003068808808076671,\n",
       "   0.014326604436638137,\n",
       "   -0.011114332376647863,\n",
       "   -0.02316530141674264,\n",
       "   0.005576953998268644,\n",
       "   0.024075555575359984,\n",
       "   0.0022063761990917955,\n",
       "   0.015263242962730121,\n",
       "   0.002628523051887801,\n",
       "   0.023349991989065122,\n",
       "   -0.02108095315005169,\n",
       "   -0.0076448136651023815,\n",
       "   0.02741315291519353,\n",
       "   -0.005557166188323932,\n",
       "   0.004224765409741216,\n",
       "   -0.023455527596318613,\n",
       "   0.0004159629293576489,\n",
       "   -0.020869878210254565,\n",
       "   0.0035453730695431717,\n",
       "   -0.008706776074053835,\n",
       "   -0.009062962240993873,\n",
       "   -0.009504897136565226,\n",
       "   0.010870278374491242,\n",
       "   0.005534079633952987,\n",
       "   0.0028198083190951405,\n",
       "   0.0007519488850920375,\n",
       "   0.0011419398482359834,\n",
       "   -0.0014354638409156528,\n",
       "   0.01322506594213599,\n",
       "   -0.01606796058277144,\n",
       "   0.003931241417061546,\n",
       "   -0.034352189406961955,\n",
       "   -0.022848692732337098,\n",
       "   -0.007710774118127714,\n",
       "   -0.0019639718019789252,\n",
       "   0.004894263844966904,\n",
       "   -0.0035882472010281942,\n",
       "   0.013759345192546046,\n",
       "   -0.038125126016159465,\n",
       "   0.010190886034293198,\n",
       "   -0.007196282677662369,\n",
       "   0.01699140599380357,\n",
       "   -0.016569259839499466,\n",
       "   0.007737158019941087,\n",
       "   -0.0012557216180631514,\n",
       "   0.005138317381462257,\n",
       "   0.011674995994532561,\n",
       "   -0.007486508391577074,\n",
       "   -0.04899540345932817,\n",
       "   -0.03065840590018837,\n",
       "   -0.003020987491274836,\n",
       "   -0.028600442000972066,\n",
       "   0.008337397723376477,\n",
       "   -0.015909654377923597,\n",
       "   -0.01261163358930201,\n",
       "   -0.008535279548113745,\n",
       "   0.004386368496370219,\n",
       "   0.017980813254844834,\n",
       "   -0.03530201918546873,\n",
       "   0.01400999482091006,\n",
       "   0.0026318210978221314,\n",
       "   0.0008970618002570486,\n",
       "   0.02941834994210563,\n",
       "   0.01809954197715818,\n",
       "   0.03501179300589275,\n",
       "   -0.013455926829232754,\n",
       "   0.018310615054310232,\n",
       "   0.014577254065002149,\n",
       "   -0.02561902896543348,\n",
       "   -0.014102339175748765,\n",
       "   -0.017294823425794324,\n",
       "   0.01361423210275806,\n",
       "   0.006084849346865329,\n",
       "   -0.005313112186167311,\n",
       "   -0.003058914903104315,\n",
       "   0.0061508097998906625,\n",
       "   -0.011892664697892005,\n",
       "   0.02828383145392145,\n",
       "   0.019141716110503654,\n",
       "   -0.009623625858878573,\n",
       "   0.015091745505467494,\n",
       "   0.037016990032804856,\n",
       "   0.02641055440173748,\n",
       "   0.021753748001397268,\n",
       "   -0.0068466921369297225,\n",
       "   -0.016318609279812917,\n",
       "   -0.03218868990049216,\n",
       "   -0.02139756183445723,\n",
       "   -0.023956826853046637,\n",
       "   -0.04910094092922673,\n",
       "   -0.012176295251260587,\n",
       "   0.020790726970475716,\n",
       "   0.0029401859479603345,\n",
       "   -0.018033580127149042,\n",
       "   -0.005725364901160326,\n",
       "   0.008311013355901837,\n",
       "   -0.046093145388858586,\n",
       "   -0.02645013188427198,\n",
       "   0.008330802097169085,\n",
       "   0.013930842649808674,\n",
       "   0.021067760034991834,\n",
       "   0.025394764635866648,\n",
       "   -0.010250250395449871,\n",
       "   0.002114031611422456,\n",
       "   0.00819888119111842,\n",
       "   0.030420948455561682,\n",
       "   -0.02910174125770009,\n",
       "   0.03720168246777241,\n",
       "   0.018033580127149042,\n",
       "   -0.012736958869145285,\n",
       "   0.022611235287710408,\n",
       "   -0.018033580127149042,\n",
       "   -0.0015822257208401704,\n",
       "   -0.011074755825435902,\n",
       "   0.005349390458614308,\n",
       "   0.004145612772978562,\n",
       "   0.008950830076210456,\n",
       "   0.028917050685377605,\n",
       "   -0.003337597572664182,\n",
       "   0.009109135349735764,\n",
       "   -0.010547072201233238,\n",
       "   0.02069838261563701,\n",
       "   0.00041946708861096,\n",
       "   0.025104538456290675,\n",
       "   0.028574057633497425,\n",
       "   -0.023798522510843866,\n",
       "   -0.014933440231942187,\n",
       "   0.015012593334366109,\n",
       "   0.005181191745777913,\n",
       "   0.04224105753988223,\n",
       "   -0.0038355987834578767,\n",
       "   0.0020942435686471094,\n",
       "   -0.005478013551561278,\n",
       "   0.017162903451066195,\n",
       "   -0.006718069509644021,\n",
       "   -0.015527084774831454,\n",
       "   0.005870477990948313,\n",
       "   0.0013192085948453957,\n",
       "   0.005464821367823958,\n",
       "   -4.498707312893459e-05,\n",
       "   -0.01122646454143128,\n",
       "   0.012730362311615355,\n",
       "   -0.00924105625578643,\n",
       "   -0.012710574501670645,\n",
       "   -0.018244653204301093,\n",
       "   -0.004251149311554588,\n",
       "   -0.028442135796124222,\n",
       "   -0.027967220906870836,\n",
       "   0.00965660678388314,\n",
       "   -0.034985410501063186,\n",
       "   -0.005672596631872313,\n",
       "   -0.023534680698742535,\n",
       "   0.03258444982467655,\n",
       "   -0.006635619059777671,\n",
       "   -0.014775135889739418,\n",
       "   0.016213073672559426,\n",
       "   -0.013488907754237322,\n",
       "   0.00017273387524204984,\n",
       "   -0.006223366344784656,\n",
       "   0.02340275886136933,\n",
       "   -0.007070957863480363,\n",
       "   0.02506496283640125,\n",
       "   -0.013944033902223457,\n",
       "   0.001169973122262472,\n",
       "   -0.02282230836486246,\n",
       "   0.006579552511724694,\n",
       "   0.003690485926500524,\n",
       "   -0.009702778961302495,\n",
       "   0.028494904531073503,\n",
       "   -0.0014890566798949068,\n",
       "   -0.012677594507988612,\n",
       "   -0.023323607621590484,\n",
       "   -0.009478513700413123,\n",
       "   -0.01638456926717698,\n",
       "   0.0013150859792198245,\n",
       "   0.011912453439159254,\n",
       "   0.20199721236862928,\n",
       "   0.014128723543223405,\n",
       "   0.005886967987789329,\n",
       "   0.018811914311038257,\n",
       "   0.0256058377130187,\n",
       "   0.04429902330174361,\n",
       "   0.004851389480651247,\n",
       "   0.015645812565822263,\n",
       "   -0.02838936706117494,\n",
       "   0.0050657608365682625,\n",
       "   -0.009584050238989148,\n",
       "   -0.010487707840076564,\n",
       "   0.012473117057043952,\n",
       "   0.009353188420569848,\n",
       "   0.00884529353763443,\n",
       "   -0.03780851733175393,\n",
       "   -0.01400999482091006,\n",
       "   -0.03234699238004986,\n",
       "   -0.004280831492132924,\n",
       "   -0.021635019279083924,\n",
       "   -0.005197681742618929,\n",
       "   -0.012670997950458684,\n",
       "   0.0015096692923614941,\n",
       "   -0.013152509397241996,\n",
       "   -0.002504847144257643,\n",
       "   0.005464821367823958,\n",
       "   -0.015263242962730121,\n",
       "   0.020724766983111653,\n",
       "   0.02048730953848496,\n",
       "   0.026159905704696007,\n",
       "   -0.01298101193997937,\n",
       "   -0.022888268352226522,\n",
       "   0.017123325968531698,\n",
       "   0.002783530046648144,\n",
       "   0.00511523129275258,\n",
       "   0.0032666901671526707,\n",
       "   0.0015170898375060783,\n",
       "   -0.000894588265806301,\n",
       "   0.01763781834031958,\n",
       "   0.017400360895692888,\n",
       "   -0.01400999482091006,\n",
       "   0.012835898850191383,\n",
       "   0.002249250563407452,\n",
       "   0.02275634651485332,\n",
       "   -0.005055866931595907,\n",
       "   0.003951029692667527,\n",
       "   ...],\n",
       "  [0.012758627224051765,\n",
       "   0.008232850982422588,\n",
       "   0.007329005639361753,\n",
       "   0.015679749492618363,\n",
       "   0.004578172452229219,\n",
       "   0.033926943106962565,\n",
       "   -0.009411779690762807,\n",
       "   -0.01002089262390793,\n",
       "   -0.01502478940953689,\n",
       "   -0.005858619982760291,\n",
       "   -0.009988144526621596,\n",
       "   0.015077186551459553,\n",
       "   -0.014775904615218837,\n",
       "   -0.012594886737620088,\n",
       "   -0.0003659590616729459,\n",
       "   0.002510135417136064,\n",
       "   0.02179707879201487,\n",
       "   -0.008665125307808644,\n",
       "   0.012241208590779332,\n",
       "   -0.0004177418799124028,\n",
       "   -0.03314099249738099,\n",
       "   0.00463056959415188,\n",
       "   0.04343696895222962,\n",
       "   -0.0006340834388166848,\n",
       "   -0.0008948395115332975,\n",
       "   -0.005835696407792119,\n",
       "   -0.007826775693659171,\n",
       "   0.005278980150908345,\n",
       "   0.0006999888332655093,\n",
       "   0.003916662842822743,\n",
       "   0.014710408420646167,\n",
       "   -0.0008109227450677178,\n",
       "   0.008056011443340899,\n",
       "   -0.025399360180285577,\n",
       "   -0.0093135353989038,\n",
       "   -0.0060649326231351575,\n",
       "   -0.023146297047450463,\n",
       "   -0.021626790176242118,\n",
       "   0.006785388900789299,\n",
       "   -0.003572808519808185,\n",
       "   0.01433053123718277,\n",
       "   -0.02108972249833336,\n",
       "   -0.01688487723758122,\n",
       "   -0.005802948310505783,\n",
       "   0.009424878743412817,\n",
       "   -0.003919937373154591,\n",
       "   -0.007754729972761495,\n",
       "   -0.045192262496428624,\n",
       "   -0.013832761648546662,\n",
       "   0.004096776912236279,\n",
       "   0.026525891746703136,\n",
       "   0.025215971580540196,\n",
       "   0.0034450911266560003,\n",
       "   -0.009758908776939867,\n",
       "   0.004008357142695434,\n",
       "   0.008344194326931604,\n",
       "   -0.015889336197663764,\n",
       "   0.015876238076336376,\n",
       "   0.02737734041350262,\n",
       "   -0.027560729013248002,\n",
       "   -0.008959857252063044,\n",
       "   -0.006471007911898574,\n",
       "   -0.029342222525392272,\n",
       "   0.0009496924347895165,\n",
       "   -0.01616442095992708,\n",
       "   0.025268367791140237,\n",
       "   -0.02495398680224951,\n",
       "   0.010145335021066957,\n",
       "   0.00013928140784572965,\n",
       "   -0.011756537123470615,\n",
       "   0.02246513885906897,\n",
       "   0.023735760935959263,\n",
       "   -0.0036121061434195257,\n",
       "   0.00682468652440064,\n",
       "   0.018744963202980313,\n",
       "   -0.02797990428598405,\n",
       "   -0.012889618681874488,\n",
       "   0.02051335673115195,\n",
       "   0.009523123035271823,\n",
       "   0.0167800829537359,\n",
       "   -0.015247476098554926,\n",
       "   -0.022216254064750918,\n",
       "   -0.023866753325104604,\n",
       "   0.017566035425962714,\n",
       "   0.0059142916550148,\n",
       "   0.019661908064352467,\n",
       "   -0.00187646129461086,\n",
       "   0.01983219854277046,\n",
       "   -0.006785388900789299,\n",
       "   -0.03177867239532753,\n",
       "   0.004702614849388245,\n",
       "   0.02847767387462016,\n",
       "   0.0011846594856669346,\n",
       "   -0.00930043634625379,\n",
       "   -0.020225173847561246,\n",
       "   0.027901309970083988,\n",
       "   -0.006477557438223579,\n",
       "   0.016950373432153894,\n",
       "   -0.006857434621686975,\n",
       "   -0.005468918742640043,\n",
       "   -0.003477839223942336,\n",
       "   0.02680097650896645,\n",
       "   -0.010499014099230333,\n",
       "   -0.009496923998649183,\n",
       "   -0.03437231648499863,\n",
       "   -0.025700643047848915,\n",
       "   -0.010806845096134743,\n",
       "   -0.017828020204253395,\n",
       "   0.028084698569829373,\n",
       "   0.012883069621210792,\n",
       "   -0.01494619416231421,\n",
       "   0.010564509362480386,\n",
       "   0.02500638487549479,\n",
       "   -0.0023906050483054634,\n",
       "   -0.0136755711541013,\n",
       "   0.017015869626726562,\n",
       "   0.024678903902631437,\n",
       "   0.01679318293770853,\n",
       "   -0.020212075726233854,\n",
       "   -0.01735644872091731,\n",
       "   0.01305990822896986,\n",
       "   -0.0010078201794159018,\n",
       "   -0.00869787247377236,\n",
       "   -0.022910512237105038,\n",
       "   -0.001688160200875741,\n",
       "   -0.010466266001943999,\n",
       "   -0.02374886091993189,\n",
       "   -0.020356166236706587,\n",
       "   0.015404666593000287,\n",
       "   -0.005252781579947015,\n",
       "   0.011704140912870573,\n",
       "   0.011317713737420862,\n",
       "   0.024587209602758744,\n",
       "   -0.015928634286936416,\n",
       "   -0.018889055576098282,\n",
       "   0.007833325685645485,\n",
       "   -0.005639208289735416,\n",
       "   0.000958698149901726,\n",
       "   0.0021187966790192367,\n",
       "   0.002253063365665771,\n",
       "   0.0055638880385058915,\n",
       "   0.027193951813757236,\n",
       "   -0.0008457174820191217,\n",
       "   -0.010164984065703283,\n",
       "   0.03143809516378203,\n",
       "   0.00865857531582233,\n",
       "   -0.017199258226471947,\n",
       "   0.007866072851609202,\n",
       "   0.006680595082605287,\n",
       "   -0.008128057629899885,\n",
       "   -0.00311106155879026,\n",
       "   -0.02970899972488304,\n",
       "   0.01003399167655794,\n",
       "   0.004001807616370429,\n",
       "   0.006300718364803201,\n",
       "   0.014225737884660068,\n",
       "   -0.004165547637140797,\n",
       "   0.014134043584787377,\n",
       "   -0.035053474673380124,\n",
       "   -0.00932663445155381,\n",
       "   0.011278416579470832,\n",
       "   0.01675388484843588,\n",
       "   0.018509178392634888,\n",
       "   0.01616442095992708,\n",
       "   0.03693976060672447,\n",
       "   0.01486759891509153,\n",
       "   0.02228175025932359,\n",
       "   0.011396308984643543,\n",
       "   -0.014094745495514727,\n",
       "   0.007846424738295495,\n",
       "   -0.0063825881423577304,\n",
       "   0.03652058719663367,\n",
       "   -0.017487441110062653,\n",
       "   0.00873717056304501,\n",
       "   0.004047654766306775,\n",
       "   0.005852070456435286,\n",
       "   0.0009537859469503085,\n",
       "   -0.0032043933566595303,\n",
       "   -0.013426686359783245,\n",
       "   -0.012365650987938358,\n",
       "   -0.02669618222512113,\n",
       "   -0.02415493620869531,\n",
       "   -0.0047353629466745815,\n",
       "   0.015260575151204936,\n",
       "   -0.004506126731331542,\n",
       "   0.0056981544923217714,\n",
       "   0.0047091643757132505,\n",
       "   0.007270059436775397,\n",
       "   0.014160241690087397,\n",
       "   -0.025268367791140237,\n",
       "   0.0019959915470261335,\n",
       "   0.0017536562790330845,\n",
       "   0.005501666374265068,\n",
       "   0.014081646442864717,\n",
       "   -0.6505589802050101,\n",
       "   0.0005530321192221267,\n",
       "   -0.017906614520153456,\n",
       "   -0.0014507370776264458,\n",
       "   0.03204065717361822,\n",
       "   0.004689515796738235,\n",
       "   0.021325507308678784,\n",
       "   0.026394901220203035,\n",
       "   0.001423719990497481,\n",
       "   -0.015915536165609027,\n",
       "   0.0021482697803124145,\n",
       "   0.02120761490350607,\n",
       "   -0.003206030621825454,\n",
       "   0.0018977474879977817,\n",
       "   -0.016609792475317908,\n",
       "   -0.008475186716076947,\n",
       "   -0.016478801948817803,\n",
       "   -0.011075378935089123,\n",
       "   0.011062279882439113,\n",
       "   -0.009536222087921833,\n",
       "   -0.019006947981270995,\n",
       "   0.01495929321496422,\n",
       "   -0.006166451445326012,\n",
       "   0.024037043803522597,\n",
       "   0.007178365136902705,\n",
       "   0.0062188481215873625,\n",
       "   0.004948225113374452,\n",
       "   -0.02729874609760256,\n",
       "   -0.03227644570925412,\n",
       "   0.024403821003013363,\n",
       "   -0.005491842317608216,\n",
       "   0.010217381207625944,\n",
       "   0.003815143787801234,\n",
       "   -0.006670770560287124,\n",
       "   0.05072012790731634,\n",
       "   -0.02361786853078655,\n",
       "   -0.010970585582566422,\n",
       "   0.013000962026383504,\n",
       "   -0.008625827218535994,\n",
       "   0.020172777636961205,\n",
       "   0.01681938104300855,\n",
       "   0.0009505111255801421,\n",
       "   0.023172495152750484,\n",
       "   0.012778275337365471,\n",
       "   -0.007564791381029797,\n",
       "   -0.006628198406343937,\n",
       "   0.04663317505173691,\n",
       "   -0.006130428352046519,\n",
       "   -0.006968777500534682,\n",
       "   -0.02431212670314067,\n",
       "   0.04878144203808146,\n",
       "   -0.00024253998434645312,\n",
       "   -0.0023316586128884526,\n",
       "   -0.005033369886922138,\n",
       "   0.011016432732502766,\n",
       "   -0.037227945352960416,\n",
       "   0.019504717569907102,\n",
       "   -0.025202873459212804,\n",
       "   0.020133479547688553,\n",
       "   0.002824516406026789,\n",
       "   0.011573148523725232,\n",
       "   0.008547231971313312,\n",
       "   -0.011907178557252282,\n",
       "   -0.045087466349938064,\n",
       "   -0.021666086402869527,\n",
       "   0.03670397579637905,\n",
       "   -0.005858619982760291,\n",
       "   -0.014579416031500824,\n",
       "   -0.0006737904369271702,\n",
       "   -0.009234940151681118,\n",
       "   -0.0093135353989038,\n",
       "   0.03471289744183462,\n",
       "   -0.011599346629025252,\n",
       "   -0.01425193598996009,\n",
       "   0.020159677652988574,\n",
       "   -0.0011412683498948021,\n",
       "   0.02659138794127581,\n",
       "   -0.012653832940206444,\n",
       "   -0.018915253681398306,\n",
       "   0.00012833128027791034,\n",
       "   0.048807642006026726,\n",
       "   -0.014461523626328113,\n",
       "   -0.04755011805046382,\n",
       "   -0.015954832392236436,\n",
       "   0.028425277664020115,\n",
       "   -0.011140875129661795,\n",
       "   -0.026394901220203035,\n",
       "   0.0025215972046201507,\n",
       "   0.026460395552130464,\n",
       "   -0.019360627059434372,\n",
       "   0.013531479712305948,\n",
       "   0.03612761002919764,\n",
       "   0.025255269669812844,\n",
       "   -0.029054039641801567,\n",
       "   -0.008743720555031326,\n",
       "   0.011153974182311805,\n",
       "   0.016505000054117824,\n",
       "   0.04217944499666672,\n",
       "   0.0023234715885668685,\n",
       "   -0.051270293706552485,\n",
       "   0.000609113107768116,\n",
       "   -0.02991858642992844,\n",
       "   0.03002338071377376,\n",
       "   0.004466829107720202,\n",
       "   0.014697309367996157,\n",
       "   -0.005282254681240193,\n",
       "   -0.007021174642457343,\n",
       "   -0.004460279581395197,\n",
       "   0.01485449986244152,\n",
       "   -0.02418113431399533,\n",
       "   -0.0013926093912077244,\n",
       "   -0.023761959041259283,\n",
       "   -0.037778111152196564,\n",
       "   0.018482980287334867,\n",
       "   -0.015142682746032223,\n",
       "   -0.02304150276360514,\n",
       "   -0.0073879523076094186,\n",
       "   0.0136755711541013,\n",
       "   -0.0010896901898010856,\n",
       "   -0.03523686327312551,\n",
       "   0.0002083592509361024,\n",
       "   -0.025818535453021624,\n",
       "   -0.010603807451753036,\n",
       "   -0.0006881176712822783,\n",
       "   0.01740884493151735,\n",
       "   0.0034287173108434876,\n",
       "   0.02564824497460363,\n",
       "   0.009778557821576193,\n",
       "   -0.0033795951649139844,\n",
       "   0.001062673102672121,\n",
       "   0.011763087115456929,\n",
       "   0.005707979014639934,\n",
       "   0.00745344850218209,\n",
       "   0.011972674751824952,\n",
       "   -0.02186257498658754,\n",
       "   0.01886285747079826,\n",
       "   0.004293265030292982,\n",
       "   -0.010066739773844277,\n",
       "   0.00287691308228814,\n",
       "   -0.02122071302483346,\n",
       "   0.006890182253312001,\n",
       "   0.030128174997619085,\n",
       "   -0.01866636888708025,\n",
       "   -0.006641297458993947,\n",
       "   -0.012424597190524715,\n",
       "   -0.00747964660748211,\n",
       "   0.007846424738295495,\n",
       "   -0.01436982932645542,\n",
       "   -0.045611435906519435,\n",
       "   0.01972740425892514,\n",
       "   -0.018286491703616854,\n",
       "   -0.0017765798540012572,\n",
       "   -0.0017454692547115005,\n",
       "   0.02871346054761082,\n",
       "   -0.026905768930166534,\n",
       "   -0.007630287575602468,\n",
       "   0.009437977796062827,\n",
       "   -0.027717919507693367,\n",
       "   -0.027848911896838708,\n",
       "   -0.0029031116532494705,\n",
       "   0.034503308874143976,\n",
       "   0.007165265618591385,\n",
       "   -0.024613407708058765,\n",
       "   -0.003328835753818557,\n",
       "   0.0055638880385058915,\n",
       "   -0.008560331023963322,\n",
       "   -0.006500481013191752,\n",
       "   0.02871346054761082,\n",
       "   0.002469200295528145,\n",
       "   -0.02740353851880264,\n",
       "   0.012542489595697426,\n",
       "   -0.011350461834707197,\n",
       "   -0.008724071510395,\n",
       "   -0.010164984065703283,\n",
       "   0.00995539642933526,\n",
       "   0.032486030551654284,\n",
       "   -0.01423883693731008,\n",
       "   -0.008357293379581614,\n",
       "   -0.0002613701053614978,\n",
       "   -0.004329287657911165,\n",
       "   0.01852227837660752,\n",
       "   -0.018247193614344202,\n",
       "   -0.004326012661918007,\n",
       "   0.021535094013724186,\n",
       "   0.006415336239644066,\n",
       "   -0.007040823221432358,\n",
       "   0.046999952251227675,\n",
       "   0.011219470376884476,\n",
       "   0.0021941169302487603,\n",
       "   0.008894361057490374,\n",
       "   0.03426752406379855,\n",
       "   0.013963754037692004,\n",
       "   -0.010774096998848409,\n",
       "   0.022111459780905597,\n",
       "   0.01501169035688688,\n",
       "   -0.01431743218453276,\n",
       "   0.0040607538189567855,\n",
       "   -0.0032764388447265513,\n",
       "   0.01438292837910543,\n",
       "   0.025163575369940155,\n",
       "   0.01089199033534374,\n",
       "   0.023709562830659243,\n",
       "   0.00528552967723335,\n",
       "   -0.008953307260076728,\n",
       "   0.01372796829602396,\n",
       "   -0.03801389596254199,\n",
       "   0.0019747052372238844,\n",
       "   -0.028399079558720095,\n",
       "   0.020749143404142613,\n",
       "   0.012476993401124755,\n",
       "   0.005514765892576388,\n",
       "   -0.04115770585144924,\n",
       "   -0.005750551634244433,\n",
       "   -0.030416356018564548,\n",
       "   -0.01148145422385254,\n",
       "   0.007977416196118218,\n",
       "   0.007067021792393689,\n",
       "   0.019557115643152385,\n",
       "   0.010361471718098677,\n",
       "   -0.015116483709409583,\n",
       "   0.009424878743412817,\n",
       "   0.010079838826494287,\n",
       "   0.02433832480844069,\n",
       "   -0.002004178571347717,\n",
       "   -0.03416272791730799,\n",
       "   0.011429057081929879,\n",
       "   0.0011036081078647127,\n",
       "   -0.024377622897713343,\n",
       "   -0.020238273831533874,\n",
       "   -0.02004178524781586,\n",
       "   -0.022190055959450897,\n",
       "   0.019124840386443707,\n",
       "   0.01374106734867397,\n",
       "   0.01121292038489816,\n",
       "   -0.009189093001744773,\n",
       "   0.002387330285142961,\n",
       "   -0.0013254759314691295,\n",
       "   -0.03149049137438207,\n",
       "   -0.01427813502658273,\n",
       "   -0.0016267576348791893,\n",
       "   0.008900910118154068,\n",
       "   0.04920061917346275,\n",
       "   -0.003030828813571,\n",
       "   -0.030547348407709892,\n",
       "   -0.018915253681398306,\n",
       "   0.01628231336509979,\n",
       "   0.027901309970083988,\n",
       "   0.012228108606806702,\n",
       "   0.004221219309395305,\n",
       "   -0.016963471553481282,\n",
       "   -0.009948847368671566,\n",
       "   -0.020893233914615346,\n",
       "   -0.02858246815846548,\n",
       "   0.03400553928550787,\n",
       "   0.00018379823617901451,\n",
       "   0.01018463311033961,\n",
       "   -0.005354300402137868,\n",
       "   -0.008606178173899668,\n",
       "   0.029185032030946908,\n",
       "   0.006752640803502964,\n",
       "   0.025713741169176303,\n",
       "   0.004113150960879446,\n",
       "   0.005023545364603975,\n",
       "   0.004240867888370321,\n",
       "   0.012490093385097385,\n",
       "   0.006700244127241612,\n",
       "   -0.005115240130137977,\n",
       "   -0.009844053084826243,\n",
       "   0.013963754037692004,\n",
       "   -0.01672768674313586,\n",
       "   0.025373162074985557,\n",
       "   -0.023355883752495865,\n",
       "   0.0004261335623797272,\n",
       "   0.011258767534834506,\n",
       "   -0.0273511423082026,\n",
       "   -0.008920559162790394,\n",
       "   -0.030128174997619085,\n",
       "   -0.01550945994552299,\n",
       "   -0.0014703858894321161,\n",
       "   0.0035171368475536767,\n",
       "   -0.01793281262545348,\n",
       "   -0.01090508938799375,\n",
       "   0.02871346054761082,\n",
       "   0.03227644570925412,\n",
       "   -0.010066739773844277,\n",
       "   0.008612728165885984,\n",
       "   -0.01491999605701419,\n",
       "   0.014487721731628133,\n",
       "   -0.009457626840699153,\n",
       "   0.029001641568556287,\n",
       "   0.027036761319311875,\n",
       "   -0.004558523873254203,\n",
       "   -0.016491900070145196,\n",
       "   -0.023735760935959263,\n",
       "   -0.0142650350426101,\n",
       "   -0.019950090947943172,\n",
       "   0.011992322865138658,\n",
       "   -0.03618000623979768,\n",
       "   0.00986370212946257,\n",
       "   -0.00837694242421794,\n",
       "   0.00839004147686795,\n",
       "   -0.018247193614344202,\n",
       "   -0.001419626478336689,\n",
       "   0.0053772239771060415,\n",
       "   -0.006493931486866747,\n",
       "   -0.0038053194983137264,\n",
       "   -0.005786574261862616,\n",
       "   -0.008553781963299627,\n",
       "   -0.02363096851475918,\n",
       "   -0.024678903902631437,\n",
       "   -0.00927423824095377,\n",
       "   0.01148145422385254,\n",
       "   0.01430433313188275,\n",
       "   -0.02866106247436554,\n",
       "   0.0010291063728028235,\n",
       "   -0.01494619416231421,\n",
       "   -0.0029751571413164915,\n",
       "   -0.02118141679820605,\n",
       "   0.017173060121171927,\n",
       "   0.006199199076951037,\n",
       "   0.0028965618940938105,\n",
       "   -0.019714306137597747,\n",
       "   -0.0016021966783297651,\n",
       "   0.003995258090045424,\n",
       "   0.08362533559435191,\n",
       "   -0.013053359168306165,\n",
       "   -0.0042146697830703,\n",
       "   0.0205526548204246,\n",
       "   0.025071881070067463,\n",
       "   -0.0005440264041099172,\n",
       "   -0.02427282861386802,\n",
       "   -0.006346565514739547,\n",
       "   -0.002623116026811005,\n",
       "   0.016570496248690496,\n",
       "   -0.012765176284715461,\n",
       "   -0.0050530189315584634,\n",
       "   0.015142682746032223,\n",
       "   0.010256678365575976,\n",
       "   0.01607272666005439,\n",
       "   0.01082649414077107,\n",
       "   -0.006019085007537501,\n",
       "   -0.030180571208219126,\n",
       "   -0.013230197776065232,\n",
       "   -0.02610671833661233,\n",
       "   -0.0236571666200592,\n",
       "   0.000928406241402595,\n",
       "   0.004944950117381295,\n",
       "   0.012142964298920325,\n",
       "   0.009077749657235757,\n",
       "   0.024967086786222142,\n",
       "   0.023355883752495865,\n",
       "   0.01881045939755298,\n",
       "   0.027141555603157196,\n",
       "   -0.003389419687232147,\n",
       "   -0.005341201349487858,\n",
       "   0.006356389571396399,\n",
       "   0.0022907237241111883,\n",
       "   -0.005681780443678604,\n",
       "   -0.006936029868909657,\n",
       "   -0.004607645553522397,\n",
       "   -0.006680595082605287,\n",
       "   0.003053752388539173,\n",
       "   0.01425193598996009,\n",
       "   -0.009117047746508407,\n",
       "   0.00015125489890183103,\n",
       "   0.0015391567307519623,\n",
       "   0.005691604965996766,\n",
       "   -0.044065730930011066,\n",
       "   0.002293998487273691,\n",
       "   0.022569931280269052,\n",
       "   0.000998814522511356,\n",
       "   0.041943656461030814,\n",
       "   0.00774818044643649,\n",
       "   -0.0015997405477502244,\n",
       "   0.01556185708744565,\n",
       "   0.01625611525979977,\n",
       "   0.009025353446635714,\n",
       "   -0.002791768308740454,\n",
       "   -0.025923329736866948,\n",
       "   0.011357011826693512,\n",
       "   -0.0006836148428300054,\n",
       "   0.00014859412472846583,\n",
       "   0.008868162952190353,\n",
       "   -0.03869505415092348,\n",
       "   -0.009090848709885767,\n",
       "   -0.03460810129534406,\n",
       "   -0.01307300728161987,\n",
       "   -0.012201910501506682,\n",
       "   0.002144995017149912,\n",
       "   -0.026342503146957755,\n",
       "   0.011383209931993533,\n",
       "   0.015299872309154966,\n",
       "   -0.023185595136723112,\n",
       "   0.009785106882239887,\n",
       "   0.011442156134579889,\n",
       "   -0.023932249519677276,\n",
       "   -0.020709845314869964,\n",
       "   -0.0017700302112609247,\n",
       "   0.02533386398571291,\n",
       "   -0.005089041559176647,\n",
       "   0.010158434073716967,\n",
       "   -0.0038642657009000823,\n",
       "   -0.002670600674743929,\n",
       "   0.019530915675207126,\n",
       "   0.0068967322452983156,\n",
       "   0.001113432513767548,\n",
       "   -0.024914690575622102,\n",
       "   -0.0010004519040926073,\n",
       "   -0.0020336516726408953,\n",
       "   0.007283158489425407,\n",
       "   0.028241889064274734,\n",
       "   -0.018129301209171493,\n",
       "   0.011448706126566205,\n",
       "   0.01615132097595445,\n",
       "   0.03232884191985416,\n",
       "   0.003661228056518374,\n",
       "   0.015050987514836913,\n",
       "   0.0035990068579388602,\n",
       "   0.018954551770670954,\n",
       "   -0.0014368191595628187,\n",
       "   0.0077612794990865,\n",
       "   0.009189093001744773,\n",
       "   0.005357575398131026,\n",
       "   -0.02052645671512458,\n",
       "   0.030232967418819166,\n",
       "   -0.003296087656532222,\n",
       "   0.006500481013191752,\n",
       "   -0.016059626676081757,\n",
       "   0.01873186508165292,\n",
       "   -0.01980600043747044,\n",
       "   -0.01916413847571636,\n",
       "   0.03838067316203276,\n",
       "   -0.004453730055070192,\n",
       "   0.008213201937786262,\n",
       "   0.006474282442230421,\n",
       "   -0.004601096027197392,\n",
       "   -0.0023578571838497827,\n",
       "   -0.011435607073916193,\n",
       "   -0.027560729013248002,\n",
       "   0.022635427474841725,\n",
       "   0.0038773649863807474,\n",
       "   0.006667496029955277,\n",
       "   -0.019976289053243192,\n",
       "   0.009627916387794524,\n",
       "   0.027744119475638627,\n",
       "   -0.013832761648546662,\n",
       "   0.01970120615362512,\n",
       "   -0.014134043584787377,\n",
       "   -0.007335555631348068,\n",
       "   -0.010767547006862093,\n",
       "   0.004794309614922247,\n",
       "   0.0008825590332585854,\n",
       "   0.0015014964887218728,\n",
       "   0.0040574792886249374,\n",
       "   0.01680628105903592,\n",
       "   0.01798521069869876,\n",
       "   -0.030678340796855233,\n",
       "   -0.023945349503649904,\n",
       "   -0.01435673027380541,\n",
       "   -0.011206370392911846,\n",
       "   0.014134043584787377,\n",
       "   0.010079838826494287,\n",
       "   0.009516573043285508,\n",
       "   -0.01801140880399878,\n",
       "   -0.0017634805685205921,\n",
       "   -0.017605333515235362,\n",
       "   -0.03235503816250894,\n",
       "   -0.008566881015949638,\n",
       "   0.009778557821576193,\n",
       "   -0.03266941915139966,\n",
       "   -0.005937215229982973,\n",
       "   -0.03002338071377376,\n",
       "   -0.0035400604225218494,\n",
       "   0.04013596670623176,\n",
       "   -0.0030701264371823405,\n",
       "   -0.000575137003399674,\n",
       "   -0.01272587912676543,\n",
       "   0.015850039971036355,\n",
       "   -0.010715150796262052,\n",
       "   -0.016295413349072422,\n",
       "   -0.010911638448657446,\n",
       "   0.003821693314126239,\n",
       "   0.009097398701872081,\n",
       "   0.018941451786698326,\n",
       "   0.010151885013053273,\n",
       "   0.0034352668371684927,\n",
       "   0.015928634286936416,\n",
       "   0.01121292038489816,\n",
       "   0.012463894348474745,\n",
       "   0.0006991701424748837,\n",
       "   -0.002187567403923755,\n",
       "   -0.006071482149460163,\n",
       "   -0.028163292885729434,\n",
       "   0.0076892337781888235,\n",
       "   0.01372796829602396,\n",
       "   0.012889618681874488,\n",
       "   0.000380900357776771,\n",
       "   0.014710408420646167,\n",
       "   0.0032584275309174597,\n",
       "   0.01809000311989884,\n",
       "   0.020565752941751992,\n",
       "   -0.01305990822896986,\n",
       "   -0.008291797185008944,\n",
       "   -0.017867316430880808,\n",
       "   -0.006271244797848713,\n",
       "   -0.008245950035072598,\n",
       "   -0.02803230049658409,\n",
       "   0.011298065624107156,\n",
       "   0.032486030551654284,\n",
       "   0.007034273695107353,\n",
       "   0.036965960574669735,\n",
       "   0.01881045939755298,\n",
       "   -0.005655582338378583,\n",
       "   -0.011298065624107156,\n",
       "   0.010590708399103026,\n",
       "   -0.0015538932813985511,\n",
       "   0.017801820236308136,\n",
       "   -0.0023087350379202795,\n",
       "   -0.0037954949759955634,\n",
       "   -0.0010151884547391964,\n",
       "   0.006042009048166984,\n",
       "   -0.042991595574193554,\n",
       "   -0.03096652368044594,\n",
       "   0.004306364082942992,\n",
       "   0.024547911513486096,\n",
       "   -0.005590086143805913,\n",
       "   0.004872904862144928,\n",
       "   -0.0029162107058994806,\n",
       "   0.025726841153148935,\n",
       "   0.03764711876305123,\n",
       "   -0.013767265453973991,\n",
       "   -0.010833043201434764,\n",
       "   -0.0161906190652271,\n",
       "   0.0028589017684790483,\n",
       "   -0.01093128749329377,\n",
       "   -0.016635992443263168,\n",
       "   -0.01860087269250758,\n",
       "   -0.04034555527392241,\n",
       "   0.00493840059105629,\n",
       "   0.00117401633076581,\n",
       "   -0.01799830882002615,\n",
       "   0.0385902617297234,\n",
       "   -0.012201910501506682,\n",
       "   -0.007558241854704792,\n",
       "   -0.028320483380174795,\n",
       "   -0.021102820619660748,\n",
       "   -0.0030095425037687507,\n",
       "   -0.010996783687866442,\n",
       "   0.02991858642992844,\n",
       "   -0.023932249519677276,\n",
       "   0.005452544693996875,\n",
       "   -0.0032322291927867845,\n",
       "   0.003962509992759089,\n",
       "   -0.008986055357363064,\n",
       "   -0.013754166401323981,\n",
       "   0.03460810129534406,\n",
       "   -0.004551974346929198,\n",
       "   0.011795835212743265,\n",
       "   -0.011180172287611825,\n",
       "   -0.017015869626726562,\n",
       "   0.020696745330897333,\n",
       "   -0.025229071564512824,\n",
       "   -0.02923742824154695,\n",
       "   0.015365368503727637,\n",
       "   0.03793529978399669,\n",
       "   -0.00839004147686795,\n",
       "   -0.017592233531262734,\n",
       "   0.01860087269250758,\n",
       "   0.010459716009957683,\n",
       "   0.02923742824154695,\n",
       "   0.00040771278675999533,\n",
       "   0.014553217926200804,\n",
       "   -0.01875806318695294,\n",
       "   0.026290106936357714,\n",
       "   -0.027875110002138728,\n",
       "   0.007348654683998078,\n",
       "   -0.021338607292651413,\n",
       "   0.014605615068123464,\n",
       "   0.026263908831057694,\n",
       "   0.011998872857124972,\n",
       "   -0.012195360509520366,\n",
       "   -0.0004916295532255751,\n",
       "   -0.020814637736070046,\n",
       "   0.005966688331276151,\n",
       "   0.001094602450960167,\n",
       "   0.020408562447306627,\n",
       "   -0.03806629217314203,\n",
       "   0.0007511576188600811,\n",
       "   0.014697309367996157,\n",
       "   0.022687825548087005,\n",
       "   -0.0021106096546976527,\n",
       "   0.00448320315636337,\n",
       "   0.00463056959415188,\n",
       "   0.02803230049658409,\n",
       "   -0.015823841865736335,\n",
       "   -0.015286773256504956,\n",
       "   0.0050399194132471434,\n",
       "   0.016963471553481282,\n",
       "   0.009516573043285508,\n",
       "   0.004633844124483727,\n",
       "   -0.018469880303362236,\n",
       "   -0.02059195290969725,\n",
       "   -0.0045716229259042136,\n",
       "   0.0058979176063716325,\n",
       "   0.015050987514836913,\n",
       "   0.005783299731530768,\n",
       "   0.0004359579682825623,\n",
       "   -0.02112901872496077,\n",
       "   0.004712439371706408,\n",
       "   -0.03576083282970687,\n",
       "   -0.005842245934117124,\n",
       "   -0.0002202303983674176,\n",
       "   -0.008324545282295278,\n",
       "   0.0007126786860393662,\n",
       "   -0.010060190713180582,\n",
       "   0.00039829772625247297,\n",
       "   -0.009667214477067176,\n",
       "   0.029394618735992313,\n",
       "   0.012732428187429125,\n",
       "   0.00837694242421794,\n",
       "   0.0073879523076094186,\n",
       "   0.02740353851880264,\n",
       "   -0.04718334085097306,\n",
       "   0.01545706280360033,\n",
       "   -0.02006798335311588,\n",
       "   0.017788722114980747,\n",
       "   -0.041498286808285226,\n",
       "   0.015823841865736335,\n",
       "   0.008049462382677204,\n",
       "   -0.016112022886681798,\n",
       "   0.004018181665013597,\n",
       "   -0.015784543776463683,\n",
       "   -0.025805435469048996,\n",
       "   -0.04899103060577211,\n",
       "   -0.01689797535890861,\n",
       "   -0.0037889454496705583,\n",
       "   -0.0007446079761197485,\n",
       "   0.007964317143468208,\n",
       "   -0.008049462382677204,\n",
       "   -0.018535376497934908,\n",
       "   0.01980600043747044,\n",
       "   -8.673107426601917e-06,\n",
       "   -0.01609892476535441,\n",
       "   0.012968213929097168,\n",
       "   -0.0012714416407958724,\n",
       "   -0.01498549225158686,\n",
       "   -0.006922930350598337,\n",
       "   -0.008141156682549895,\n",
       "   0.022530635053641643,\n",
       "   0.007211112768527731,\n",
       "   -0.010649654601689382,\n",
       "   -0.0007499295535703108,\n",
       "   -0.03167387997412745,\n",
       "   -0.006657671507637114,\n",
       "   0.002554345301906486,\n",
       "   -0.011736889010156909,\n",
       "   -0.006431710288287234,\n",
       "   -0.013387388270510595,\n",
       "   -0.005583536617480908,\n",
       "   -0.010158434073716967,\n",
       "   -0.006212298595262357,\n",
       "   0.026604487925248436,\n",
       "   0.003170007994207271,\n",
       "   -0.004905652493769954,\n",
       "   0.011095027979725449,\n",
       "   0.0180376069092988,\n",
       "   0.0008604540908733744,\n",
       "   0.0015170517883667512,\n",
       "   0.022609229369541704,\n",
       "   -0.011016432732502766,\n",
       "   -0.035027278430725346,\n",
       "   -0.007630287575602468,\n",
       "   -0.013282594917987894,\n",
       "   -0.03620620620774294,\n",
       "   -0.005485292791283211,\n",
       "   0.03908803131835951,\n",
       "   0.011959574767852322,\n",
       "   -0.023224893225995764,\n",
       "   0.0013623174245009295,\n",
       "   0.009333184443540124,\n",
       "   -0.05221343667322466,\n",
       "   0.0074141508785707496,\n",
       "   0.015339170398427616,\n",
       "   0.014107844548164737,\n",
       "   0.04320118041659372,\n",
       "   -0.008468636724090631,\n",
       "   -0.0054066975440605295,\n",
       "   0.023185595136723112,\n",
       "   0.00954277207990815,\n",
       "   0.019661908064352467,\n",
       "   -0.011612446612997882,\n",
       "   0.0010643104842533722,\n",
       "   0.017487441110062653,\n",
       "   -0.009503473990635497,\n",
       "   0.029342222525392272,\n",
       "   -0.01928203088088907,\n",
       "   -0.0311237141748913,\n",
       "   -0.00208768607972948,\n",
       "   0.009248039204331128,\n",
       "   0.024639607676004024,\n",
       "   -0.007925019985518178,\n",
       "   0.007663035672888803,\n",
       "   -0.005924116177332963,\n",
       "   0.00373327377741605,\n",
       "   -0.007879172835581831,\n",
       "   0.0023251090865634475,\n",
       "   0.001479391546336662,\n",
       "   0.02052645671512458,\n",
       "   0.01867946887105288,\n",
       "   -0.0434631651948844,\n",
       "   -0.009280787301617464,\n",
       "   0.005806223306498941,\n",
       "   0.0028146918837086265,\n",
       "   0.008422789574154285,\n",
       "   0.0025134101802985667,\n",
       "   0.00015770215436162964,\n",
       "   -0.022255552154023566,\n",
       "   -0.009241490143667434,\n",
       "   -0.03232884191985416,\n",
       "   -0.01554875803479564,\n",
       "   -0.004781210096610927,\n",
       "   -0.008029813338040879,\n",
       "   -0.016518098175445216,\n",
       "   0.02661758604657583,\n",
       "   -0.0229891065530051,\n",
       "   0.018561574603234928,\n",
       "   0.005475468268965048,\n",
       "   -0.011527301373788885,\n",
       "   0.0019681557108988793,\n",
       "   -0.0038708154600557423,\n",
       "   -0.04152448305094001,\n",
       "   -0.004840156764858593,\n",
       "   -0.008861612960204037,\n",
       "   -0.03345537348627172,\n",
       "   -0.02923742824154695,\n",
       "   -0.023198693258050504,\n",
       "   -0.0021973919262419177,\n",
       "   -0.010394219815385013,\n",
       "   0.000495723065386367,\n",
       "   0.00865857531582233,\n",
       "   0.015981032360181696,\n",
       "   -0.007820225701672855,\n",
       "   0.00966066448508086,\n",
       "   0.022818817937232345,\n",
       "   -0.023709562830659243,\n",
       "   -0.007040823221432358,\n",
       "   -0.0204740586418793,\n",
       "   -0.013014061079033514,\n",
       "   0.0019337703484466199,\n",
       "   -0.007073571318718694,\n",
       "   -0.0025379712532633183,\n",
       "   -0.014042349284914686,\n",
       "   0.008285248124345248,\n",
       "   -0.0021351704948317494,\n",
       "   -0.011579698515711546,\n",
       "   0.0038053194983137264,\n",
       "   0.01945232135930706,\n",
       "   -0.0018240645019341817,\n",
       "   -0.011710689973534269,\n",
       "   -0.0019468695175119575,\n",
       "   0.20424282055544737,\n",
       "   0.010479365054594009,\n",
       "   -0.017605333515235362,\n",
       "   0.011717239965520583,\n",
       "   -0.006994976071496012,\n",
       "   0.030259165524119187,\n",
       "   0.003821693314126239,\n",
       "   0.01032872362081234,\n",
       "   -0.00390028856134892,\n",
       "   0.017028967748053955,\n",
       "   -0.019976289053243192,\n",
       "   -0.015732145703218403,\n",
       "   0.011998872857124972,\n",
       "   0.007106319416005029,\n",
       "   -0.002973519876150568,\n",
       "   -0.04883384197397198,\n",
       "   -0.013858959753846684,\n",
       "   -0.015679749492618363,\n",
       "   -0.024678903902631437,\n",
       "   -0.0036153809065820283,\n",
       "   0.008016714285390868,\n",
       "   -0.018378186003489547,\n",
       "   -0.0371755454170699,\n",
       "   -0.01370176925940132,\n",
       "   0.026656884135848477,\n",
       "   -0.008914010102126698,\n",
       "   -0.023958447624977296,\n",
       "   -0.0067264422325416335,\n",
       "   -0.003949410474447769,\n",
       "   0.02305460274757777,\n",
       "   -0.003263339792076541,\n",
       "   -0.02479679630780415,\n",
       "   -0.008514483874026977,\n",
       "   0.013171251573478877,\n",
       "   -0.001025831609640321,\n",
       "   -0.004005082146702277,\n",
       "   0.04375134994112034,\n",
       "   -0.018836657502853002,\n",
       "   0.008599629113235974,\n",
       "   0.03523686327312551,\n",
       "   -0.01797211071472613,\n",
       "   0.021574392102996838,\n",
       "   0.021639888297569507,\n",
       "   -0.011075378935089123,\n",
       "   -0.00777437901739782,\n",
       "   0.023329685647195845,\n",
       "   ...]],\n",
       " 'metadatas': [{'bathrooms': 3,\n",
       "   'bedrooms': 4,\n",
       "   'house size': 280,\n",
       "   'neighborhood': 'Central Park',\n",
       "   'price': 3500000},\n",
       "  {'bathrooms': 2,\n",
       "   'bedrooms': 3,\n",
       "   'house size': 200,\n",
       "   'neighborhood': 'Ocean View',\n",
       "   'price': 1800000}],\n",
       " 'documents': ['Modern Luxury in Central Park: Indulge in the epitome of modern luxury with this stunning 4-bedroom, 3-bathroom home in Central Park. Boasting high-end finishes, a gourmet kitchen, and panoramic views of the city skyline, this property offers a sophisticated urban lifestyle. Neighborhood: Central Park is a vibrant neighborhood known for its upscale dining, shopping, and cultural attractions. Enjoy easy access to the park for leisurely strolls and outdoor activities.',\n",
       "  'Seaside Retreat in Ocean View: Escape to this charming seaside retreat in Ocean View. With 3 bedrooms, 2 bathrooms, and a spacious deck overlooking the ocean, this home offers a perfect blend of coastal living and modern comfort. Neighborhood: Ocean View is a picturesque coastal community with sandy beaches, seafood restaurants, and a laid-back atmosphere. Experience the tranquility of seaside living in this charming neighborhood.']}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "collection.peek(2) # returns a list of the first 2 items in the collection"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f990dc5f",
   "metadata": {},
   "source": [
    "# STEP 3 - SEMANTIC SEARCH FONCTIONALITY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "id": "6f151292",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>listings_headline</th>\n",
       "      <th>listings_neighborhood</th>\n",
       "      <th>listings_price</th>\n",
       "      <th>listings_bedrooms</th>\n",
       "      <th>listings_bathrooms</th>\n",
       "      <th>listings_house_size</th>\n",
       "      <th>listings_description</th>\n",
       "      <th>listings_neighborhood_description</th>\n",
       "      <th>full_description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Modern Luxury in Central Park</td>\n",
       "      <td>Central Park</td>\n",
       "      <td>3500000</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>280</td>\n",
       "      <td>Indulge in the epitome of modern luxury with t...</td>\n",
       "      <td>Central Park is a vibrant neighborhood known f...</td>\n",
       "      <td>Modern Luxury in Central Park: Indulge in the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Seaside Retreat in Ocean View</td>\n",
       "      <td>Ocean View</td>\n",
       "      <td>1800000</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>200</td>\n",
       "      <td>Escape to this charming seaside retreat in Oce...</td>\n",
       "      <td>Ocean View is a picturesque coastal community ...</td>\n",
       "      <td>Seaside Retreat in Ocean View: Escape to this ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Historic Elegance in Heritage District</td>\n",
       "      <td>Heritage District</td>\n",
       "      <td>1200000</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>320</td>\n",
       "      <td>Step back in time with this historic 5-bedroom...</td>\n",
       "      <td>The Heritage District is a historic neighborho...</td>\n",
       "      <td>Historic Elegance in Heritage District: Step b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Contemporary Living in Skyline Heights</td>\n",
       "      <td>Skyline Heights</td>\n",
       "      <td>2800000</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>400</td>\n",
       "      <td>Experience contemporary living at its finest i...</td>\n",
       "      <td>Skyline Heights offers a vibrant urban lifesty...</td>\n",
       "      <td>Contemporary Living in Skyline Heights: Experi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Tranquil Haven in Willow Creek</td>\n",
       "      <td>Willow Creek</td>\n",
       "      <td>1500000</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>250</td>\n",
       "      <td>Find peace and serenity in this tranquil 4-bed...</td>\n",
       "      <td>Willow Creek is a quiet residential neighborho...</td>\n",
       "      <td>Tranquil Haven in Willow Creek: Find peace and...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Urban Chic in Downtown District</td>\n",
       "      <td>Downtown District</td>\n",
       "      <td>2000000</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>180</td>\n",
       "      <td>Embrace urban chic living in this stylish 3-be...</td>\n",
       "      <td>The Downtown District is a bustling urban cent...</td>\n",
       "      <td>Urban Chic in Downtown District: Embrace urban...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Rustic Charm in Maple Grove</td>\n",
       "      <td>Maple Grove</td>\n",
       "      <td>1300000</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>220</td>\n",
       "      <td>Discover the rustic charm of this 4-bedroom, 3...</td>\n",
       "      <td>Maple Grove is a quaint neighborhood with tree...</td>\n",
       "      <td>Rustic Charm in Maple Grove: Discover the rust...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Luxurious Retreat in Hillside Manor</td>\n",
       "      <td>Hillside Manor</td>\n",
       "      <td>3000000</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>350</td>\n",
       "      <td>Escape to a luxurious retreat in the prestigio...</td>\n",
       "      <td>Hillside Manor is an exclusive neighborhood kn...</td>\n",
       "      <td>Luxurious Retreat in Hillside Manor: Escape to...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Coastal Living in Seaview Heights</td>\n",
       "      <td>Seaview Heights</td>\n",
       "      <td>2200000</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>190</td>\n",
       "      <td>Experience the beauty of coastal living in thi...</td>\n",
       "      <td>Seaview Heights is a coastal community with sa...</td>\n",
       "      <td>Coastal Living in Seaview Heights: Experience ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Elegant Oasis in Rosewood Estates</td>\n",
       "      <td>Rosewood Estates</td>\n",
       "      <td>2600000</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>240</td>\n",
       "      <td>Discover an elegant oasis in the prestigious R...</td>\n",
       "      <td>Rosewood Estates is an upscale neighborhood kn...</td>\n",
       "      <td>Elegant Oasis in Rosewood Estates: Discover an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Serenity in Pinecrest Meadows</td>\n",
       "      <td>Pinecrest Meadows</td>\n",
       "      <td>1700000</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>260</td>\n",
       "      <td>Find serenity in this 5-bedroom, 4-bathroom ho...</td>\n",
       "      <td>Pinecrest Meadows is a nature lover's paradise...</td>\n",
       "      <td>Serenity in Pinecrest Meadows: Find serenity i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Modern Elegance in Silver Lake</td>\n",
       "      <td>Silver Lake</td>\n",
       "      <td>1900000</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>210</td>\n",
       "      <td>Experience modern elegance in this 3-bedroom, ...</td>\n",
       "      <td>Silver Lake is a trendy neighborhood known for...</td>\n",
       "      <td>Modern Elegance in Silver Lake: Experience mod...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Charming Cottage in Willowbrook Village</td>\n",
       "      <td>Willowbrook Village</td>\n",
       "      <td>1400000</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>150</td>\n",
       "      <td>Escape to this charming cottage in the heart o...</td>\n",
       "      <td>Willowbrook Village is a quaint village with h...</td>\n",
       "      <td>Charming Cottage in Willowbrook Village: Escap...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Luxury Living in Diamond Heights</td>\n",
       "      <td>Diamond Heights</td>\n",
       "      <td>2700000</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>230</td>\n",
       "      <td>Indulge in luxury living in this 4-bedroom, 3-...</td>\n",
       "      <td>Diamond Heights is an exclusive neighborhood k...</td>\n",
       "      <td>Luxury Living in Diamond Heights: Indulge in l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Tranquil Retreat in Cedar Valley</td>\n",
       "      <td>Cedar Valley</td>\n",
       "      <td>1600000</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>170</td>\n",
       "      <td>Find peace and tranquility in this 3-bedroom, ...</td>\n",
       "      <td>Cedar Valley is a peaceful neighborhood with h...</td>\n",
       "      <td>Tranquil Retreat in Cedar Valley: Find peace a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Urban Oasis in City Center</td>\n",
       "      <td>City Center</td>\n",
       "      <td>2500000</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>260</td>\n",
       "      <td>Discover an urban oasis in the heart of the Ci...</td>\n",
       "      <td>City Center is a vibrant urban hub with shoppi...</td>\n",
       "      <td>Urban Oasis in City Center: Discover an urban ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          listings_headline listings_neighborhood  \\\n",
       "0             Modern Luxury in Central Park          Central Park   \n",
       "1             Seaside Retreat in Ocean View            Ocean View   \n",
       "2    Historic Elegance in Heritage District     Heritage District   \n",
       "3    Contemporary Living in Skyline Heights       Skyline Heights   \n",
       "4            Tranquil Haven in Willow Creek          Willow Creek   \n",
       "5           Urban Chic in Downtown District     Downtown District   \n",
       "6               Rustic Charm in Maple Grove           Maple Grove   \n",
       "7       Luxurious Retreat in Hillside Manor        Hillside Manor   \n",
       "8         Coastal Living in Seaview Heights       Seaview Heights   \n",
       "9         Elegant Oasis in Rosewood Estates      Rosewood Estates   \n",
       "10            Serenity in Pinecrest Meadows     Pinecrest Meadows   \n",
       "11           Modern Elegance in Silver Lake           Silver Lake   \n",
       "12  Charming Cottage in Willowbrook Village   Willowbrook Village   \n",
       "13         Luxury Living in Diamond Heights       Diamond Heights   \n",
       "14         Tranquil Retreat in Cedar Valley          Cedar Valley   \n",
       "15               Urban Oasis in City Center           City Center   \n",
       "\n",
       "    listings_price  listings_bedrooms  listings_bathrooms  \\\n",
       "0          3500000                  4                   3   \n",
       "1          1800000                  3                   2   \n",
       "2          1200000                  5                   4   \n",
       "3          2800000                  6                   5   \n",
       "4          1500000                  4                   3   \n",
       "5          2000000                  3                   2   \n",
       "6          1300000                  4                   3   \n",
       "7          3000000                  5                   4   \n",
       "8          2200000                  3                   2   \n",
       "9          2600000                  4                   3   \n",
       "10         1700000                  5                   4   \n",
       "11         1900000                  3                   2   \n",
       "12         1400000                  2                   1   \n",
       "13         2700000                  4                   3   \n",
       "14         1600000                  3                   2   \n",
       "15         2500000                  4                   3   \n",
       "\n",
       "    listings_house_size                               listings_description  \\\n",
       "0                   280  Indulge in the epitome of modern luxury with t...   \n",
       "1                   200  Escape to this charming seaside retreat in Oce...   \n",
       "2                   320  Step back in time with this historic 5-bedroom...   \n",
       "3                   400  Experience contemporary living at its finest i...   \n",
       "4                   250  Find peace and serenity in this tranquil 4-bed...   \n",
       "5                   180  Embrace urban chic living in this stylish 3-be...   \n",
       "6                   220  Discover the rustic charm of this 4-bedroom, 3...   \n",
       "7                   350  Escape to a luxurious retreat in the prestigio...   \n",
       "8                   190  Experience the beauty of coastal living in thi...   \n",
       "9                   240  Discover an elegant oasis in the prestigious R...   \n",
       "10                  260  Find serenity in this 5-bedroom, 4-bathroom ho...   \n",
       "11                  210  Experience modern elegance in this 3-bedroom, ...   \n",
       "12                  150  Escape to this charming cottage in the heart o...   \n",
       "13                  230  Indulge in luxury living in this 4-bedroom, 3-...   \n",
       "14                  170  Find peace and tranquility in this 3-bedroom, ...   \n",
       "15                  260  Discover an urban oasis in the heart of the Ci...   \n",
       "\n",
       "                    listings_neighborhood_description  \\\n",
       "0   Central Park is a vibrant neighborhood known f...   \n",
       "1   Ocean View is a picturesque coastal community ...   \n",
       "2   The Heritage District is a historic neighborho...   \n",
       "3   Skyline Heights offers a vibrant urban lifesty...   \n",
       "4   Willow Creek is a quiet residential neighborho...   \n",
       "5   The Downtown District is a bustling urban cent...   \n",
       "6   Maple Grove is a quaint neighborhood with tree...   \n",
       "7   Hillside Manor is an exclusive neighborhood kn...   \n",
       "8   Seaview Heights is a coastal community with sa...   \n",
       "9   Rosewood Estates is an upscale neighborhood kn...   \n",
       "10  Pinecrest Meadows is a nature lover's paradise...   \n",
       "11  Silver Lake is a trendy neighborhood known for...   \n",
       "12  Willowbrook Village is a quaint village with h...   \n",
       "13  Diamond Heights is an exclusive neighborhood k...   \n",
       "14  Cedar Valley is a peaceful neighborhood with h...   \n",
       "15  City Center is a vibrant urban hub with shoppi...   \n",
       "\n",
       "                                     full_description  \n",
       "0   Modern Luxury in Central Park: Indulge in the ...  \n",
       "1   Seaside Retreat in Ocean View: Escape to this ...  \n",
       "2   Historic Elegance in Heritage District: Step b...  \n",
       "3   Contemporary Living in Skyline Heights: Experi...  \n",
       "4   Tranquil Haven in Willow Creek: Find peace and...  \n",
       "5   Urban Chic in Downtown District: Embrace urban...  \n",
       "6   Rustic Charm in Maple Grove: Discover the rust...  \n",
       "7   Luxurious Retreat in Hillside Manor: Escape to...  \n",
       "8   Coastal Living in Seaview Heights: Experience ...  \n",
       "9   Elegant Oasis in Rosewood Estates: Discover an...  \n",
       "10  Serenity in Pinecrest Meadows: Find serenity i...  \n",
       "11  Modern Elegance in Silver Lake: Experience mod...  \n",
       "12  Charming Cottage in Willowbrook Village: Escap...  \n",
       "13  Luxury Living in Diamond Heights: Indulge in l...  \n",
       "14  Tranquil Retreat in Cedar Valley: Find peace a...  \n",
       "15  Urban Oasis in City Center: Discover an urban ...  "
      ]
     },
     "execution_count": 254,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load property database to pandas\n",
    "df = pd.read_json('data/property_listings.json')\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eac211e1",
   "metadata": {},
   "source": [
    "### RUBRIC #3 Semantic Search of Listings Based on Buyer Preferences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "6cbec4ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "def query_db(query: str,\n",
    "             metadata: Optional[dict] = None,\n",
    "             search_string: Optional[str] = ' ',\n",
    "             n_results: int = 10\n",
    "             ):\n",
    "    \"\"\"\n",
    "    text query the database and return the n closest vector entries\n",
    "    inputs:\n",
    "    - query: the query string\n",
    "    - metadata: the metadata to filter the results\n",
    "        - Provide metadata as a dictionary with metadata keys and values as lists of values to filter for\n",
    "        - targets can be composed with the following operators:\n",
    "        e.g {'house size': {\"$gte\": 200}\n",
    "            $eq - equal to (string, int, float)\n",
    "            $ne - not equal to (string, int, float)\n",
    "            $gt - greater than (int, float)\n",
    "            $gte - greater than or equal to (int, float)\n",
    "            $lt - less than (int, float)\n",
    "            $lte - less than or equal to (int, float)\n",
    "    - search_string: the string or keyword to search in the listing description\n",
    "    - n_results: the number of results to return (int - default = 10)\n",
    "    returns:\n",
    "    - the n closest results as a list of dictionaries.\n",
    "    The results are ordered by the distance to the query in ascending order (lowest in best match)\n",
    "    Returns the ids, description, metadata and distance of the corresponding vector entries\n",
    "    \"\"\"\n",
    "    # compute the embeddings of the query for semantic search\n",
    "    query_embeddings = embeddings_model.embed_documents([query])\n",
    "    # defines the keywords for the search function\n",
    "    kwargs  = {'query_embeddings':query_embeddings, 'n_results':n_results}\n",
    "    if metadata:\n",
    "        kwargs['where'] = metadata\n",
    "    if search_string:\n",
    "        kwargs['where_document'] = {\"$contains\":search_string}\n",
    "    # serach the vector db using the defined arguments & return results\n",
    "    results = collection.query(**kwargs)\n",
    "\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "dbf62a3a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ids': [['8', '1', '11']],\n",
       " 'distances': [[0.2822312941790316, 0.2836603463282615, 0.4023989250458718]],\n",
       " 'metadatas': [[{'bathrooms': 2,\n",
       "    'bedrooms': 3,\n",
       "    'house size': 190,\n",
       "    'neighborhood': 'Seaview Heights',\n",
       "    'price': 2200000},\n",
       "   {'bathrooms': 2,\n",
       "    'bedrooms': 3,\n",
       "    'house size': 200,\n",
       "    'neighborhood': 'Ocean View',\n",
       "    'price': 1800000},\n",
       "   {'bathrooms': 2,\n",
       "    'bedrooms': 3,\n",
       "    'house size': 210,\n",
       "    'neighborhood': 'Silver Lake',\n",
       "    'price': 1900000}]],\n",
       " 'embeddings': None,\n",
       " 'documents': [['Coastal Living in Seaview Heights: Experience the beauty of coastal living in this 3-bedroom, 2-bathroom home in Seaview Heights. With ocean views, a sun-drenched deck, and beach access just steps away, this property offers a perfect seaside retreat. Neighborhood: Seaview Heights is a coastal community with sandy beaches, seaside cafes, and water sports activities. Enjoy the laid-back lifestyle and stunning ocean views in this charming neighborhood.',\n",
       "   'Seaside Retreat in Ocean View: Escape to this charming seaside retreat in Ocean View. With 3 bedrooms, 2 bathrooms, and a spacious deck overlooking the ocean, this home offers a perfect blend of coastal living and modern comfort. Neighborhood: Ocean View is a picturesque coastal community with sandy beaches, seafood restaurants, and a laid-back atmosphere. Experience the tranquility of seaside living in this charming neighborhood.',\n",
       "   'Modern Elegance in Silver Lake: Experience modern elegance in this 3-bedroom, 2-bathroom home in Silver Lake. With sleek design, high ceilings, and a rooftop deck with panoramic views of the lake, this property offers a perfect blend of style and sophistication. Neighborhood: Silver Lake is a trendy neighborhood known for its hip cafes, art galleries, and scenic lake views. Embrace the vibrant energy and creative spirit of this eclectic community.']]}"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = \"nice house with a view on the sea\"\n",
    "query_db(query=query, n_results=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "a8e8b30b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ids': [['1', '11', '3']],\n",
       " 'distances': [[0.2836603463282615, 0.4023989250458718, 0.40659283973841437]],\n",
       " 'metadatas': [[{'bathrooms': 2,\n",
       "    'bedrooms': 3,\n",
       "    'house size': 200,\n",
       "    'neighborhood': 'Ocean View',\n",
       "    'price': 1800000},\n",
       "   {'bathrooms': 2,\n",
       "    'bedrooms': 3,\n",
       "    'house size': 210,\n",
       "    'neighborhood': 'Silver Lake',\n",
       "    'price': 1900000},\n",
       "   {'bathrooms': 5,\n",
       "    'bedrooms': 6,\n",
       "    'house size': 400,\n",
       "    'neighborhood': 'Skyline Heights',\n",
       "    'price': 2800000}]],\n",
       " 'embeddings': None,\n",
       " 'documents': [['Seaside Retreat in Ocean View: Escape to this charming seaside retreat in Ocean View. With 3 bedrooms, 2 bathrooms, and a spacious deck overlooking the ocean, this home offers a perfect blend of coastal living and modern comfort. Neighborhood: Ocean View is a picturesque coastal community with sandy beaches, seafood restaurants, and a laid-back atmosphere. Experience the tranquility of seaside living in this charming neighborhood.',\n",
       "   'Modern Elegance in Silver Lake: Experience modern elegance in this 3-bedroom, 2-bathroom home in Silver Lake. With sleek design, high ceilings, and a rooftop deck with panoramic views of the lake, this property offers a perfect blend of style and sophistication. Neighborhood: Silver Lake is a trendy neighborhood known for its hip cafes, art galleries, and scenic lake views. Embrace the vibrant energy and creative spirit of this eclectic community.',\n",
       "   'Contemporary Living in Skyline Heights: Experience contemporary living at its finest in this 6-bedroom, 5-bathroom home in Skyline Heights. With sleek design, high ceilings, and a rooftop terrace with city views, this property is perfect for modern urban dwellers. Neighborhood: Skyline Heights offers a vibrant urban lifestyle with trendy cafes, art galleries, and easy access to public transportation. Enjoy the bustling energy of the city from the comfort of your stylish home.']]}"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query_db(query=query, n_results=3, metadata={'house size': {\"$gte\": 200}})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "bbad5ebb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ids': [['1']],\n",
       " 'distances': [[0.2836603463282615]],\n",
       " 'metadatas': [[{'bathrooms': 2,\n",
       "    'bedrooms': 3,\n",
       "    'house size': 200,\n",
       "    'neighborhood': 'Ocean View',\n",
       "    'price': 1800000}]],\n",
       " 'embeddings': None,\n",
       " 'documents': [['Seaside Retreat in Ocean View: Escape to this charming seaside retreat in Ocean View. With 3 bedrooms, 2 bathrooms, and a spacious deck overlooking the ocean, this home offers a perfect blend of coastal living and modern comfort. Neighborhood: Ocean View is a picturesque coastal community with sandy beaches, seafood restaurants, and a laid-back atmosphere. Experience the tranquility of seaside living in this charming neighborhood.']]}"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query_db(query=query, n_results=3, metadata={'house size': {\"$gte\": 200}}, search_string='sea')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d61d4a7",
   "metadata": {},
   "source": [
    "# STEP 4 - BUILD USER PREFERENCE INTERFACE"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd646d28",
   "metadata": {},
   "source": [
    "- prepare schema and output parser to extract key preference characteristics from customer message"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "id": "4c98cdc3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The output should be a markdown code snippet formatted in the following schema, including the leading and trailing \"```json\" and \"```\":\n",
      "\n",
      "```json\n",
      "{\n",
      "\t\"price\": string  // Did the customer indicated a maximum price for the property?                                 Provide the price value if yes, -1 if not or unknown.\n",
      "\t\"bathrooms\": string  // Did the customer indicated the number of bathrooms for the property?                                     Provide the number value if yes, -1 if not or unknown.\n",
      "\t\"bedrooms\": string  // Did the customer indicated the number of bedrooms for the property?                                     Provide the number value if yes, -1 if not or unknown.\n",
      "\t\"house size\": string  // Did the customer indicated the minimum size of the property?                                 Provide the number value if yes, -1 if not or unknown.\n",
      "}\n",
      "```\n"
     ]
    }
   ],
   "source": [
    "# Keep this cell formating as it influences the llm output with int instead of 'int' !\n",
    "price_schema = ResponseSchema(name=\"price\",\n",
    "                              description=\"Did the customer indicated a maximum price for the property? \\\n",
    "                                Provide the price value if yes, -1 if not or unknown.\")\n",
    "bathrooms_schema = ResponseSchema(name=\"bathrooms\",\n",
    "                                  description=\"Did the customer indicated the number of bathrooms for the property? \\\n",
    "                                    Provide the number value if yes, -1 if not or unknown.\")\n",
    "bedrooms_schema = ResponseSchema(name=\"bedrooms\",\n",
    "                                 description=\"Did the customer indicated the number of bedrooms for the property? \\\n",
    "                                    Provide the number value if yes, -1 if not or unknown.\")\n",
    "size_schema = ResponseSchema(name=\"house size\",\n",
    "                             description=\"Did the customer indicated the minimum size of the property? \\\n",
    "                                Provide the number value if yes, -1 if not or unknown.\")\n",
    "\n",
    "response_schemas = [price_schema, \n",
    "                    bathrooms_schema,\n",
    "                    bedrooms_schema,\n",
    "                    size_schema]\n",
    "\n",
    "output_parser = StructuredOutputParser.from_response_schemas(response_schemas)\n",
    "format_user_preferences = output_parser.get_format_instructions()\n",
    "print(format_user_preferences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "id": "41a05bd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_where_expression(dict_of_metadata):\n",
    "    \"\"\"\n",
    "    Builds a where_document for metadata search, format expected by vectorDB\n",
    "    \"\"\"\n",
    "    # eliminate all metadata keys with no value identified (-1)\n",
    "    dict_of_metadata = {k:v for k,v in dict_of_metadata.items() if v != -1}\n",
    "    # build metadata search expression using Where format expected by vectorDB\n",
    "    expression = {}\n",
    "    try:\n",
    "        max_price = dict_of_metadata['price']\n",
    "        # use less than or equal operator for price\n",
    "        dict_of_metadata['price'] = {\"$lte\": max_price}\n",
    "    except:\n",
    "        pass\n",
    "    if len(dict_of_metadata) >1:\n",
    "        # combine all metadata search expressions into a single Where expression\n",
    "        expression = {\"$and\": [{k:v} for k,v in dict_of_metadata.items()]}\n",
    "    return expression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "id": "2248b81c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prompt template for finding user preferences.\n",
    "# It uses the schema defined above\n",
    "find_preferences_template = \"\"\"\\\n",
    "For the following customer input, extract the following information:\n",
    "\n",
    "price: Did the customer indicated a maximum price for the property? \\\n",
    "Provide the price value if yes, -1 if not or unknown.\n",
    "\n",
    "bathrooms: Did the customer indicated the number of bathrooms for the property? \\\n",
    "Provide the number value if yes, -1 if not or unknown.\n",
    "\n",
    "bedrooms: Did the customer indicated the number of bedrooms for the property? \\\n",
    "Provide the number value if yes, -1 if not or unknown.\n",
    "\n",
    "House size: Did the customer indicated the minimum size of the property? \\\n",
    "Provide the number value if yes, -1 if not or unknown.\n",
    "\n",
    "customer input: {customer_input}\n",
    "\n",
    "{format_user_preferences}\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "id": "85d830a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_best_matches(customer_input, top_k=3):\n",
    "    '''   \n",
    "    takes in the customer text input indicating his preferences, find the best property matches and extract customer preference characteristics as metadata\n",
    "    returns :\n",
    "    - the top k (default = 3) best property matches using query_db function\n",
    "    - the key characteristics indicated by the customer for the preperty metadata\n",
    "\n",
    "    input:\n",
    "    - customer_input: str\n",
    "    outputs:\n",
    "    - listings_results: dictionary with property matches from query_db function\n",
    "    - output_dict: dictionary with key, value pairs representing the metadata and their extracted values\n",
    "    '''\n",
    "    pref_prompt = ChatPromptTemplate.from_template(template=find_preferences_template)\n",
    "    messages = pref_prompt.format_messages(customer_input=customer_input, \n",
    "                                  format_user_preferences=format_user_preferences)\n",
    "\n",
    "    # Use LLM to extratc the preferences from the conversation\n",
    "    response = chat_llm(messages)\n",
    "    output_dict = output_parser.parse(response.content)\n",
    "    print(f'Extracted desired preferencies: {output_dict}')\n",
    "\n",
    "    # Query the vectordb with the customer input and the key preference characteristics on metadata\n",
    "    listings_results = query_db(query=customer_input, \n",
    "                                n_results=top_k, \n",
    "                                metadata=build_where_expression(output_dict)\n",
    "                                )\n",
    "    return listings_results, output_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "id": "c0586c0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_matches(matches, threshold=0.52, df=df):\n",
    "    '''   \n",
    "    takes in the list of property matches from query_db function\n",
    "    displays the property characterics from the df dataframe\n",
    "    filter out the matches with distance above a threshold (default is 0.52)\n",
    "    inputs:\n",
    "    - property matches from query_db function\n",
    "    - threshold: float between 0 and 1. lower is more restrictive\n",
    "    - df: dataframe with the properties details (default is database)\n",
    "    outputs:\n",
    "    - matches_df: input dataframe filtered to property matches below threshold\n",
    "    '''\n",
    "    idx = matches['ids'][0]\n",
    "    idx = [int(i) for i in idx]\n",
    "    if len(idx)>0:\n",
    "        idx = [id for i,id in enumerate(idx) if matches['distances'][0][i]<=threshold]\n",
    "        \n",
    "    if len(idx)>0:\n",
    "        matches_df = df.loc[idx]\n",
    "        \n",
    "        for i, id in enumerate(idx):\n",
    "                print(YELLOW + f\"Distance {str(round(matches['distances'][0][i],3))} : {matches_df.loc[id,'listings_headline']}\" + ENDC)\n",
    "                for key in list(listings_description_model.model_fields.keys())[1:]:\n",
    "                    print(f'- {key.split(\"listings_\")[1].replace(\"_\",\" \"):<25} : {matches_df.loc[id,key]}')\n",
    "                print('-------------------------------------------------')\n",
    "    else:\n",
    "        matches_df = pd.DataFrame()\n",
    "        print(RED + \"No match found with these characteristics\" + ENDC)\n",
    "\n",
    "    return matches_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09f654d1",
   "metadata": {},
   "source": [
    "### RUBRIC #3 Semantic Search of Listings Based on Buyer Preferences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "id": "53721e66",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted desired preferencies: {'price': 2000000, 'bathrooms': 3, 'bedrooms': 3, 'house size': -1}\n",
      "\u001b[31mNo match found with these characteristics\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "customer_input = \"i am looking for a property with a view on the sea with 3 bedrooms, 3 bathrooms and maximum price of 2000000 euros\"\n",
    "\n",
    "extrated_properties, extracted_prefs = find_best_matches(customer_input)\n",
    "\n",
    "matches_df = display_matches(extrated_properties)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "id": "6a0c8d10",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted desired preferencies: {'price': 2000000, 'bathrooms': 2, 'bedrooms': 3, 'house size': -1}\n",
      "\u001b[33mDistance 0.439 : Seaside Retreat in Ocean View\u001b[0m\n",
      "- neighborhood              : Ocean View\n",
      "- price                     : 1800000\n",
      "- bedrooms                  : 3\n",
      "- bathrooms                 : 2\n",
      "- house size                : 200\n",
      "- description               : Escape to this charming seaside retreat in Ocean View. With 3 bedrooms, 2 bathrooms, and a spacious deck overlooking the ocean, this home offers a perfect blend of coastal living and modern comfort.\n",
      "- neighborhood description  : Ocean View is a picturesque coastal community with sandy beaches, seafood restaurants, and a laid-back atmosphere. Experience the tranquility of seaside living in this charming neighborhood.\n",
      "-------------------------------------------------\n",
      "\u001b[33mDistance 0.51 : Modern Elegance in Silver Lake\u001b[0m\n",
      "- neighborhood              : Silver Lake\n",
      "- price                     : 1900000\n",
      "- bedrooms                  : 3\n",
      "- bathrooms                 : 2\n",
      "- house size                : 210\n",
      "- description               : Experience modern elegance in this 3-bedroom, 2-bathroom home in Silver Lake. With sleek design, high ceilings, and a rooftop deck with panoramic views of the lake, this property offers a perfect blend of style and sophistication.\n",
      "- neighborhood description  : Silver Lake is a trendy neighborhood known for its hip cafes, art galleries, and scenic lake views. Embrace the vibrant energy and creative spirit of this eclectic community.\n",
      "-------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "customer_input = \"i am looking for a property with a view on the sea with 3 bedrooms, 2 bathrooms and maximum price of 2000000 euros\"\n",
    "\n",
    "extrated_properties, extracted_prefs = find_best_matches(customer_input)\n",
    "\n",
    "matches_df = display_matches(extrated_properties)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22607a61",
   "metadata": {},
   "source": [
    "# STEP 5 - AUGMENT AND PERSONALIZE LISTING DESCRIPTION TO CUSTOMER PREFERENCES"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b01a2f5",
   "metadata": {},
   "source": [
    "### RUBRIC #4 Logic for Searching and Augmenting Listing Descriptions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9dcb28e3",
   "metadata": {},
   "source": [
    "- summarize the preferences of the customer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "id": "66426073",
   "metadata": {},
   "outputs": [],
   "source": [
    "summarize_preferences_prompt_template = ''' \n",
    "Based on the customer input and extracted perference characterisctics below, summarize the user preferencies describing the ideal property the customer is looking for.\n",
    "- Be concise.\n",
    "- Only use what is provided to you. Do not alter or modify preferencies.\n",
    "- Do not add preferencies not mentionned by the customer as this could flaw the conclusion.\n",
    "- if you do not know, say \"I do not know\"\n",
    "customer input: {customer_input}\n",
    "extracted preferencies: {extracted_preferences}\n",
    "'''\n",
    "summ_prompt = PromptTemplate.from_template(summarize_preferences_prompt_template)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "id": "ce06e0aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "summarizing_prompt = summ_prompt.format(customer_input=query, extracted_preferences = extracted_prefs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "id": "bb6c9bdb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "The customer is looking for a 3 bedroom, 2 bathroom house with a view of the sea, with a budget of $2,000,000.\n"
     ]
    }
   ],
   "source": [
    "summarized_preferences = instruct_llm(summarizing_prompt)\n",
    "print(summarized_preferences)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1730de9a",
   "metadata": {},
   "source": [
    "- Use to augment descriptions and personalize to the expectations "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 380,
   "id": "91cb7944",
   "metadata": {},
   "outputs": [],
   "source": [
    "augmentation_prompt_template = '''\n",
    "You are a real estate agent with great style to emphasize the property features to the customer preferences.\n",
    "Based on the description of what the customer is looking for, the property description and property characteristics, rewrite a personalized, catchy, brief on the property enhancing those features.\n",
    "- Be concise.\n",
    "- Stick to what is provided to you. Do not alter or modify the customer preferences.\n",
    "- only emphasize confirmed characteristics\n",
    "- Do not invent preferences not mentionned by the customer since this could negatively flaw the conclusion.\n",
    "- if you do not know, say \"I do not know\"\n",
    "property descriptiont: {property_description}\n",
    "customer preferences: {summarized_preferences}\n",
    "'''\n",
    "aug_prompt = PromptTemplate.from_template(augmentation_prompt_template)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 403,
   "id": "6ae6cc6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def augment_properties(properties,summarized_preferences, df):\n",
    "    '''   \n",
    "    Takes in best property matches, summary of buyer preferences and initial property description\n",
    "    Generates an augmented description for each selected property enhancing the buyer's features\n",
    "    Display the augmented descriptions together with the exact characteristics of the properties\n",
    "    inputs:\n",
    "    - best property matches from find_best_matches function\n",
    "    - summarized buyer preferences (str)\n",
    "    - dataframe with detailed characteristics of the bast matches\n",
    "    outputs:\n",
    "    - dataframe with detailed characteristics of the bast matches and personalized, augmented, description using an LLM\n",
    "    '''\n",
    "    idx = df.index.to_list()\n",
    "    \n",
    "    keys = ['listings_neighborhood',\n",
    "        'listings_price',\n",
    "        'listings_bedrooms',\n",
    "        'listings_bathrooms',\n",
    "        'listings_house_size',\n",
    "        #'listings_augmented_description',\n",
    "        ]\n",
    "    \n",
    "    if len(idx) > 0:\n",
    "        for id in idx:\n",
    "            full_description = df.loc[id, 'full_description']\n",
    "            augmentation_prompt = aug_prompt.format(property_description=full_description, summarized_preferences = summarized_preferences)\n",
    "            augmented_desc = instruct_llm(augmentation_prompt)\n",
    "            df.loc[id, 'augmented_description'] = augmented_desc\n",
    "        \n",
    "        for i, id in enumerate(idx):\n",
    "            print(YELLOW + f\"Distance {str(round(properties['distances'][0][i],3))} : {df.loc[id,'listings_headline']}\" + ENDC)\n",
    "            for key in keys:\n",
    "                print(f'- {key.split(\"listings_\")[1].replace(\"_\",\" \"):<25} : {df.loc[id,key]}')\n",
    "            \n",
    "            display(Markdown('<span style=\"color:cyan\">'+f'augmented description : {df.loc[id,\"augmented_description\"]}'+'</span>'))\n",
    "            print('-------------------------------------------------')\n",
    "\n",
    "    else:\n",
    "        print(RED + \"No match found with these characteristics\" + ENDC)\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8244870",
   "metadata": {},
   "source": [
    "### RUBRIC #5 Use of LLM for Generating Personalized Descriptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "id": "6a80ea32",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mDistance 0.439 : Seaside Retreat in Ocean View\u001b[0m\n",
      "- neighborhood              : Ocean View\n",
      "- price                     : 1800000\n",
      "- bedrooms                  : 3\n",
      "- bathrooms                 : 2\n",
      "- house size                : 200\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "<span style=\"color:cyan\">augmented description : \n",
       "Welcome to your dream seaside retreat in Ocean View! This charming home boasts 3 bedrooms and 2 bathrooms, perfect for your family's needs. Step out onto the spacious deck and take in the breathtaking view of the ocean. Imagine waking up to the sound of waves crashing and enjoying your morning coffee with a stunning view. With a budget of $2,000,000, this home offers the perfect blend of coastal living and modern comfort. Located in the picturesque neighborhood of Ocean View, you'll have access to sandy beaches, delicious seafood restaurants, and a laid-back atmosphere. Don't miss out on the opportunity to experience the tranquility of seaside living in this charming community. </span>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------\n",
      "\u001b[33mDistance 0.51 : Modern Elegance in Silver Lake\u001b[0m\n",
      "- neighborhood              : Silver Lake\n",
      "- price                     : 1900000\n",
      "- bedrooms                  : 3\n",
      "- bathrooms                 : 2\n",
      "- house size                : 210\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "<span style=\"color:cyan\">augmented description : \n",
       "\"Experience modern elegance in this stunning 3-bedroom, 2-bathroom home located in the trendy Silver Lake neighborhood. With sleek design and high ceilings, this property exudes sophistication. But the real showstopper? The rooftop deck with panoramic views of the lake, perfect for enjoying the vibrant energy and creative spirit of this eclectic community. And with a budget of $2,000,000, this is the perfect opportunity to own a piece of Silver Lake's hip cafes, art galleries, and scenic lake views. Don't miss out on this modern gem!\"</span>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "augmented_df = augment_properties(extrated_properties, summarized_preferences, matches_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 407,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>listings_headline</th>\n",
       "      <th>listings_neighborhood</th>\n",
       "      <th>listings_price</th>\n",
       "      <th>listings_bedrooms</th>\n",
       "      <th>listings_bathrooms</th>\n",
       "      <th>listings_house_size</th>\n",
       "      <th>listings_description</th>\n",
       "      <th>listings_neighborhood_description</th>\n",
       "      <th>full_description</th>\n",
       "      <th>augmented_description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Seaside Retreat in Ocean View</td>\n",
       "      <td>Ocean View</td>\n",
       "      <td>1800000</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>200</td>\n",
       "      <td>Escape to this charming seaside retreat in Oce...</td>\n",
       "      <td>Ocean View is a picturesque coastal community ...</td>\n",
       "      <td>Seaside Retreat in Ocean View: Escape to this ...</td>\n",
       "      <td>\\nWelcome to your dream seaside retreat in Oce...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Modern Elegance in Silver Lake</td>\n",
       "      <td>Silver Lake</td>\n",
       "      <td>1900000</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>210</td>\n",
       "      <td>Experience modern elegance in this 3-bedroom, ...</td>\n",
       "      <td>Silver Lake is a trendy neighborhood known for...</td>\n",
       "      <td>Modern Elegance in Silver Lake: Experience mod...</td>\n",
       "      <td>\\n\"Experience modern elegance in this stunning...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 listings_headline listings_neighborhood  listings_price  \\\n",
       "1    Seaside Retreat in Ocean View            Ocean View         1800000   \n",
       "11  Modern Elegance in Silver Lake           Silver Lake         1900000   \n",
       "\n",
       "    listings_bedrooms  listings_bathrooms  listings_house_size  \\\n",
       "1                   3                   2                  200   \n",
       "11                  3                   2                  210   \n",
       "\n",
       "                                 listings_description  \\\n",
       "1   Escape to this charming seaside retreat in Oce...   \n",
       "11  Experience modern elegance in this 3-bedroom, ...   \n",
       "\n",
       "                    listings_neighborhood_description  \\\n",
       "1   Ocean View is a picturesque coastal community ...   \n",
       "11  Silver Lake is a trendy neighborhood known for...   \n",
       "\n",
       "                                     full_description  \\\n",
       "1   Seaside Retreat in Ocean View: Escape to this ...   \n",
       "11  Modern Elegance in Silver Lake: Experience mod...   \n",
       "\n",
       "                                augmented_description  \n",
       "1   \\nWelcome to your dream seaside retreat in Oce...  \n",
       "11  \\n\"Experience modern elegance in this stunning...  "
      ]
     },
     "execution_count": 407,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "augmented_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ad13efc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "c791203f",
   "metadata": {},
   "source": [
    "# EXTRA: WORKFLOW WITH TOOL + FUNCTION CALLING TO EXTRACT AND REPOND WITH PERSONALIZED DESCRIPTIONS"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "103a39ab",
   "metadata": {},
   "source": [
    "- define tool to search for best match in property DB using pydantic structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 682,
   "id": "98407af0",
   "metadata": {},
   "outputs": [],
   "source": [
    "class GetBestPropertyMatchToolInput(BaseModel):\n",
    "    \"\"\"Call this function to find the properties best matching customer preferences\"\"\"\n",
    "    query: str = Field(description=\"summary of the property's characteristics the customer looks for\")\n",
    "    metadata: Optional[dict] = Field(description=\"dictionary of metadata and value pairs of the target properties such as price, number of bedrooms, number of bathrooms, house surface, neighborhood\")\n",
    "    search_string: Optional[str] = Field(description=\"keyword to search for in the property description\")\n",
    "    n_results: Optional[NonNegativeInt] = Field(description=\"the number of property match to return - default = 3\")\n",
    "\n",
    "\n",
    "class GetBestPropertyMatchTool(BaseTool):\n",
    "    name = \"query_db\"\n",
    "    description = \"Call this function to find the properties best matching customer preferences\"\n",
    "\n",
    "    def _run(self, query:str, metadata: Optional[dict] = None, search_string: Optional[str] = None, n_results: Optional[NonNegativeInt] = 3):\n",
    "        \"\"\"\n",
    "        Run the tool and return the best match\n",
    "        \"\"\"\n",
    "        # Query the database\n",
    "        best_matches = query_db(query, metadata, search_string, n_results)  #TODO build_where_expression(metadata) ??\n",
    "        return best_matches\n",
    "    \n",
    "    def _arun(self, query:str, metadata: Optional[dict] = None, search_string: Optional[str] = None, n_results: Optional[NonNegativeInt] = 3):\n",
    "        \"\"\"\n",
    "        Run the tool and return the best match\n",
    "        \"\"\"\n",
    "        raise NotImplementedError('This tool does not support async')\n",
    "    \n",
    "    args_schema: Type[BaseModel] = GetBestPropertyMatchToolInput\n",
    "\n",
    "\n",
    "# Declare tools and functions\n",
    "tools = [GetBestPropertyMatchTool()]\n",
    "functions = [format_tool_to_openai_function(tool_name) for tool_name in tools]\n",
    "\n",
    "# Set up a parser + inject instructions into the prompt template.\n",
    "parser = PydanticOutputParser(pydantic_object=GetBestPropertyMatchToolInput)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 683,
   "id": "60bd27d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GetBestPropertyMatchTool(name='query_db', description='Call this function to find the properties best matching customer preferences', args_schema=<class '__main__.GetBestPropertyMatchToolInput'>, return_direct=False, verbose=False, callbacks=None, callback_manager=None, tags=None, metadata=None, handle_tool_error=False)"
      ]
     },
     "execution_count": 683,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tools[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 475,
   "id": "442a63a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'name': 'query_db',\n",
       "  'description': 'Call this function to find the properties best matching customer preferences',\n",
       "  'parameters': {'description': 'Call this function to find the properties best matching customer preferences',\n",
       "   'properties': {'query': {'description': \"summary of the property's characteristics the customer looks for\",\n",
       "     'title': 'Query',\n",
       "     'type': 'string'},\n",
       "    'metadata': {'anyOf': [{'type': 'object'}, {'type': 'null'}],\n",
       "     'description': 'dictionary of metadata and value pairs of the target properties such as price, number of bedrooms, number of bathrooms, house surface, neighborhood',\n",
       "     'title': 'Metadata'},\n",
       "    'search_string': {'anyOf': [{'type': 'string'}, {'type': 'null'}],\n",
       "     'description': 'keyword to search for in the property description',\n",
       "     'title': 'Search String'},\n",
       "    'n_results': {'anyOf': [{'minimum': 0, 'type': 'integer'},\n",
       "      {'type': 'null'}],\n",
       "     'description': 'the number of property match to return - default = 3',\n",
       "     'title': 'N Results'}},\n",
       "   'required': ['query', 'metadata', 'search_string', 'n_results'],\n",
       "   'title': 'GetBestPropertyMatchToolInput',\n",
       "   'type': 'object'}}]"
      ]
     },
     "execution_count": 475,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "functions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbb2f17a",
   "metadata": {},
   "source": [
    "- generate function calling based on customer preferences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 498,
   "id": "4f18d04d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AIMessage(content='', additional_kwargs={'function_call': {'name': 'query_db', 'arguments': '{\"query\":\"3 bedrooms, lake view\",\"metadata\":null,\"search_string\":null,\"n_results\":3}'}})"
      ]
     },
     "execution_count": 498,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "customer_input = \"i am looking for a house with 3 bedrooms and a view on the lake?\"\n",
    "ai_response = chat_llm.predict_messages([HumanMessage(content=customer_input)], functions=functions)\n",
    "# similar to response = chat_llm.invoke(\"i am looking for a house with 3 bedrooms and a view on the lake?\", functions=functions)\n",
    "ai_response"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23dd294a",
   "metadata": {},
   "source": [
    "- extract function calling output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 514,
   "id": "78c11d9e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "query_db\n",
      "{'query': '3 bedrooms, lake view', 'metadata': None, 'search_string': None, 'n_results': 3}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'ids': [['11', '1', '6']],\n",
       " 'distances': [[0.3404558433476212, 0.3557057768479667, 0.3824061701253278]],\n",
       " 'metadatas': [[{'bathrooms': 2,\n",
       "    'bedrooms': 3,\n",
       "    'house size': 210,\n",
       "    'neighborhood': 'Silver Lake',\n",
       "    'price': 1900000},\n",
       "   {'bathrooms': 2,\n",
       "    'bedrooms': 3,\n",
       "    'house size': 200,\n",
       "    'neighborhood': 'Ocean View',\n",
       "    'price': 1800000},\n",
       "   {'bathrooms': 3,\n",
       "    'bedrooms': 4,\n",
       "    'house size': 220,\n",
       "    'neighborhood': 'Maple Grove',\n",
       "    'price': 1300000}]],\n",
       " 'embeddings': None,\n",
       " 'documents': [['Modern Elegance in Silver Lake: Experience modern elegance in this 3-bedroom, 2-bathroom home in Silver Lake. With sleek design, high ceilings, and a rooftop deck with panoramic views of the lake, this property offers a perfect blend of style and sophistication. Neighborhood: Silver Lake is a trendy neighborhood known for its hip cafes, art galleries, and scenic lake views. Embrace the vibrant energy and creative spirit of this eclectic community.',\n",
       "   'Seaside Retreat in Ocean View: Escape to this charming seaside retreat in Ocean View. With 3 bedrooms, 2 bathrooms, and a spacious deck overlooking the ocean, this home offers a perfect blend of coastal living and modern comfort. Neighborhood: Ocean View is a picturesque coastal community with sandy beaches, seafood restaurants, and a laid-back atmosphere. Experience the tranquility of seaside living in this charming neighborhood.',\n",
       "   'Rustic Charm in Maple Grove: Discover the rustic charm of this 4-bedroom, 3-bathroom home in Maple Grove. With exposed wooden beams, a cozy fireplace, and a spacious backyard with a fire pit, this property offers a warm and inviting atmosphere. Neighborhood: Maple Grove is a quaint neighborhood with tree-lined streets, local cafes, and boutique shops. Enjoy a sense of community and small-town charm in this cozy enclave.']]}"
      ]
     },
     "execution_count": 514,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if ai_response.content=='':\n",
    "    tool_name = ai_response.additional_kwargs['function_call']['name']\n",
    "    tool = [tools[i] for i, tool in enumerate(tools) if tool.name==tool_name][0]\n",
    "    _args = json.loads(ai_response.additional_kwargs['function_call']['arguments'])\n",
    "print(tool.name)\n",
    "print(_args)\n",
    "extrated_properties = tool(_args)\n",
    "extrated_properties"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5bbe83b7",
   "metadata": {},
   "source": [
    "- narrow down the list using threshold parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 515,
   "id": "53bc2513",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mDistance 0.34 : Modern Elegance in Silver Lake\u001b[0m\n",
      "- neighborhood              : Silver Lake\n",
      "- price                     : 1900000\n",
      "- bedrooms                  : 3\n",
      "- bathrooms                 : 2\n",
      "- house size                : 210\n",
      "- description               : Experience modern elegance in this 3-bedroom, 2-bathroom home in Silver Lake. With sleek design, high ceilings, and a rooftop deck with panoramic views of the lake, this property offers a perfect blend of style and sophistication.\n",
      "- neighborhood description  : Silver Lake is a trendy neighborhood known for its hip cafes, art galleries, and scenic lake views. Embrace the vibrant energy and creative spirit of this eclectic community.\n",
      "-------------------------------------------------\n",
      "\u001b[33mDistance 0.356 : Seaside Retreat in Ocean View\u001b[0m\n",
      "- neighborhood              : Ocean View\n",
      "- price                     : 1800000\n",
      "- bedrooms                  : 3\n",
      "- bathrooms                 : 2\n",
      "- house size                : 200\n",
      "- description               : Escape to this charming seaside retreat in Ocean View. With 3 bedrooms, 2 bathrooms, and a spacious deck overlooking the ocean, this home offers a perfect blend of coastal living and modern comfort.\n",
      "- neighborhood description  : Ocean View is a picturesque coastal community with sandy beaches, seafood restaurants, and a laid-back atmosphere. Experience the tranquility of seaside living in this charming neighborhood.\n",
      "-------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>listings_headline</th>\n",
       "      <th>listings_neighborhood</th>\n",
       "      <th>listings_price</th>\n",
       "      <th>listings_bedrooms</th>\n",
       "      <th>listings_bathrooms</th>\n",
       "      <th>listings_house_size</th>\n",
       "      <th>listings_description</th>\n",
       "      <th>listings_neighborhood_description</th>\n",
       "      <th>full_description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Modern Elegance in Silver Lake</td>\n",
       "      <td>Silver Lake</td>\n",
       "      <td>1900000</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>210</td>\n",
       "      <td>Experience modern elegance in this 3-bedroom, ...</td>\n",
       "      <td>Silver Lake is a trendy neighborhood known for...</td>\n",
       "      <td>Modern Elegance in Silver Lake: Experience mod...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Seaside Retreat in Ocean View</td>\n",
       "      <td>Ocean View</td>\n",
       "      <td>1800000</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>200</td>\n",
       "      <td>Escape to this charming seaside retreat in Oce...</td>\n",
       "      <td>Ocean View is a picturesque coastal community ...</td>\n",
       "      <td>Seaside Retreat in Ocean View: Escape to this ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 listings_headline listings_neighborhood  listings_price  \\\n",
       "11  Modern Elegance in Silver Lake           Silver Lake         1900000   \n",
       "1    Seaside Retreat in Ocean View            Ocean View         1800000   \n",
       "\n",
       "    listings_bedrooms  listings_bathrooms  listings_house_size  \\\n",
       "11                  3                   2                  210   \n",
       "1                   3                   2                  200   \n",
       "\n",
       "                                 listings_description  \\\n",
       "11  Experience modern elegance in this 3-bedroom, ...   \n",
       "1   Escape to this charming seaside retreat in Oce...   \n",
       "\n",
       "                    listings_neighborhood_description  \\\n",
       "11  Silver Lake is a trendy neighborhood known for...   \n",
       "1   Ocean View is a picturesque coastal community ...   \n",
       "\n",
       "                                     full_description  \n",
       "11  Modern Elegance in Silver Lake: Experience mod...  \n",
       "1   Seaside Retreat in Ocean View: Escape to this ...  "
      ]
     },
     "execution_count": 515,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matches_df = display_matches(extrated_properties,0.37)\n",
    "matches_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6d0c1f6",
   "metadata": {},
   "source": [
    "- generate augmented personalized descriptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 516,
   "id": "3d2d11e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mDistance 0.34 : Modern Elegance in Silver Lake\u001b[0m\n",
      "- neighborhood              : Silver Lake\n",
      "- price                     : 1900000\n",
      "- bedrooms                  : 3\n",
      "- bathrooms                 : 2\n",
      "- house size                : 210\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "<span style=\"color:cyan\">augmented description : \n",
       "\"Welcome to your dream home in Silver Lake! This stunning 3-bedroom, 2-bathroom property boasts modern elegance with its sleek design, high ceilings, and a rooftop deck offering breathtaking views of the lake. Immerse yourself in the trendy and vibrant neighborhood, known for its hip cafes, art galleries, and scenic lake views. Don't miss the opportunity to experience the perfect blend of style and sophistication in this sought-after community. Come see it for yourself!\"</span>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------\n",
      "\u001b[33mDistance 0.356 : Seaside Retreat in Ocean View\u001b[0m\n",
      "- neighborhood              : Ocean View\n",
      "- price                     : 1800000\n",
      "- bedrooms                  : 3\n",
      "- bathrooms                 : 2\n",
      "- house size                : 200\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "<span style=\"color:cyan\">augmented description : \n",
       "Welcome to your dream seaside retreat in Ocean View! This charming home boasts 3 bedrooms and a stunning view of the ocean from its spacious deck. Immerse yourself in the perfect blend of coastal living and modern comfort. Located in the picturesque neighborhood of Ocean View, you'll have access to sandy beaches, delicious seafood restaurants, and a laid-back atmosphere. Don't miss out on the opportunity to experience the tranquility of seaside living in this charming community. Come see it for yourself today!</span>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "augmented_df = augment_properties(extrated_properties, customer_input, matches_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bbce338f",
   "metadata": {},
   "source": [
    "- Generate final LLM response in natural language"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 518,
   "id": "879c763f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "I found some properties that match your preferences:\n",
       "\n",
       "1. **Modern Elegance in Silver Lake**\n",
       "   - **Neighborhood:** Silver Lake\n",
       "   - **Price:** $1,900,000\n",
       "   - **Bedrooms:** 3\n",
       "   - **Bathrooms:** 2\n",
       "   - **House Size:** 210 sqft\n",
       "   - **Description:** Welcome to your dream home in Silver Lake! This stunning 3-bedroom, 2-bathroom property boasts modern elegance with its sleek design, high ceilings, and a rooftop deck offering breathtaking views of the lake. Immerse yourself in the trendy and vibrant neighborhood, known for its hip cafes, art galleries, and scenic lake views. Don't miss the opportunity to experience the perfect blend of style and sophistication in this sought-after community. Come see it for yourself!\n",
       "\n",
       "2. **Seaside Retreat in Ocean View**\n",
       "   - **Neighborhood:** Ocean View\n",
       "   - **Price:** $1,800,000\n",
       "   - **Bedrooms:** 3\n",
       "   - **Bathrooms:** 2\n",
       "   - **House Size:** 200 sqft\n",
       "   - **Description:** Welcome to your dream seaside retreat in Ocean View! This charming home boasts 3 bedrooms and a stunning view of the ocean from its spacious deck. Immerse yourself in the perfect blend of coastal living and modern comfort. Located in the picturesque neighborhood of Ocean View, you'll have access to sandy beaches, delicious seafood restaurants, and a laid-back atmosphere. Don't miss out on the opportunity to experience the tranquility of seaside living in this charming community. Come see it for yourself today!"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "final_output = chat_llm.predict_messages([ \n",
    "    HumanMessage(content=customer_input),\n",
    "    ai_response,\n",
    "    FunctionMessage(name=tool.name, content=str(extrated_properties)),\n",
    "    AIMessage(content=str((augmented_df\n",
    "                           .drop(['listings_description','listings_neighborhood_description','full_description'], axis=1)  # we keep the augmented description and the metadata characteristics\n",
    "                           .to_dict(orient='dict')\n",
    "                           )\n",
    "                           ))\n",
    "    ], \n",
    "    functions=functions\n",
    ")\n",
    "display(Markdown(final_output.content))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07b43e0f",
   "metadata": {},
   "source": [
    "# FULL AUTOMATIZED WORKFLOW with automated preference collection using tagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 549,
   "id": "5ecdabdc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chains import create_tagging_chain, create_tagging_chain_pydantic\n",
    "from enum import Enum\n",
    "from pydantic.v1 import BaseModel, Field  # required due to bug issue in later pydantic version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 664,
   "id": "4165dcb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "chat_llm = ChatOpenAI(model='gpt-3.5-turbo-1106', temperature=0.1, max_tokens=3500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 684,
   "id": "1469ce4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "class PreferenceDetails(BaseModel):\n",
    "    price: NonNegativeInt = Field(description=\"This is the maximum price the customer can afford to buy a property.\",default=0)\n",
    "    #planned_price: NonNegativeInt = Field(description=\"This is the price amount the customer would prefer to spend to buy a property.\",default=0)\n",
    "    #neighborhood: str = Field(description=\"This is the neighborhood where the customer wants to buy a property\",default='')\n",
    "    bedrooms: NonNegativeInt = Field(description=\"This is the number of desired bedrooms for the property\",default=0)\n",
    "    bathrooms: NonNegativeInt = Field(description=\"This is the number of desired bathrooms for the property\",default=0)\n",
    "    size: NonNegativeInt = Field(#enum=[\"spanish\", \"english\", \"french\", \"german\", \"italian\"]\n",
    "                                description=\"This is the minimum desired size of the property in square meters\",default=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 685,
   "id": "5a2f07ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PreferenceDetails(price=0, bedrooms=0, bathrooms=0, size=0)"
      ]
     },
     "execution_count": 685,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test\n",
    "tt = PreferenceDetails()\n",
    "tt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 686,
   "id": "f1b5ddd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "chain = create_tagging_chain_pydantic(PreferenceDetails, chat_llm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 687,
   "id": "1a2d3bff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PreferenceDetails(price=2000000, bedrooms=3, bathrooms=3, size=0)"
      ]
     },
     "execution_count": 687,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test\n",
    "customer_input = \"i am looking for a property with a view on the sea with 3 bedrooms, 3 bathrooms and maximum price of 2000000 euros\"\n",
    "res = chain.run(customer_input)\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 688,
   "id": "77c7fb09",
   "metadata": {},
   "outputs": [],
   "source": [
    "# check which metadata field are still unknown and empty\n",
    "def check_what_is_empty(user_preferences):\n",
    "    ask_for = []\n",
    "    # Check if fields are empty\n",
    "    for field, value in user_preferences.dict().items():\n",
    "        if value in [None, \"\", 0]:  # You can add other 'empty' conditions as per your requirements\n",
    "            print(RED + f\"Field '{field}' is empty.\" + ENDC)\n",
    "            ask_for.append(f'{field}')\n",
    "    return ask_for\n",
    "\n",
    "## checking the response and adding it\n",
    "def add_non_empty_details(current_details: PreferenceDetails, new_details: PreferenceDetails):\n",
    "    non_empty_details = {k: v for k, v in new_details.dict().items() if v not in [None, 0, \"\"]}\n",
    "    updated_details = current_details.copy(update=non_empty_details)\n",
    "    return updated_details\n",
    "\n",
    "\n",
    "def ask_for_info(ask_for = ['maximum price','number of bedrooms', 'neighborhood']):\n",
    "\n",
    "    # prompt template 1\n",
    "    first_prompt = ChatPromptTemplate.from_template(\n",
    "        \"You are an real estate agent. Below is are some things to ask the customer for in a conversation way to understand his preferences. you should only ask one question at a time even if you don't get all the info \\\n",
    "        don't ask as a list! Don't greet the user! Don't say Hi. Explain you need to get some info to identify the best property match.\\n\\n \\\n",
    "        ### ask_for list: {ask_for}\"  #If the ask_for list is empty then thank them and ask how you can help them \n",
    "    )\n",
    "\n",
    "    # info_gathering_chain\n",
    "    info_gathering_chain = LLMChain(llm=chat_llm, prompt=first_prompt)\n",
    "    ai_chat = info_gathering_chain.run(ask_for=ask_for)\n",
    "    return ai_chat\n",
    "\n",
    "\n",
    "def filter_response(text_input, user_preferences ):\n",
    "    tagging_chain = create_tagging_chain_pydantic(PreferenceDetails, chat_llm)\n",
    "    res = tagging_chain.run(text_input)\n",
    "    # add filtered info to the\n",
    "    user_preferences = add_non_empty_details(user_preferences,res)\n",
    "    ask_for = check_what_is_empty(user_preferences)\n",
    "    return user_preferences, ask_for\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 689,
   "id": "ef89d693",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mField 'price' is empty.\u001b[0m\n",
      "\u001b[31mField 'bedrooms' is empty.\u001b[0m\n",
      "\u001b[31mField 'bathrooms' is empty.\u001b[0m\n",
      "\u001b[31mField 'size' is empty.\u001b[0m\n",
      "\u001b[33mWhat is your budget for the property you are looking for? Answer adding what your answer is about, e.g. \"4 bedrooms\"\u001b[0m\n",
      "\u001b[31mField 'bedrooms' is empty.\u001b[0m\n",
      "\u001b[31mField 'bathrooms' is empty.\u001b[0m\n",
      "\u001b[31mField 'size' is empty.\u001b[0m\n",
      "\u001b[33mHow many bedrooms are you looking for in a property? Answer adding what your answer is about, e.g. \"4 bedrooms\"\u001b[0m\n",
      "\u001b[31mField 'bathrooms' is empty.\u001b[0m\n",
      "\u001b[31mField 'size' is empty.\u001b[0m\n",
      "\u001b[33mHow many bathrooms are you looking for in your ideal property? Answer adding what your answer is about, e.g. \"4 bedrooms\"\u001b[0m\n",
      "\u001b[31mField 'size' is empty.\u001b[0m\n",
      "\u001b[33mWhat size of property are you looking for? Answer adding what your answer is about, e.g. \"4 bedrooms\"\u001b[0m\n",
      "Thank you ! Everything has been gathered, moving now to next screening phase\n",
      "\u001b[33mBesides these criteria, describe the property you would love to buy ?\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "user_preferences = PreferenceDetails()\n",
    "ask_for = check_what_is_empty(user_preferences)\n",
    "\n",
    "# while loop until all info are collected\n",
    "while ask_for:\n",
    "    ai_response = ask_for_info(ask_for) + ' Answer adding what your answer is about, e.g. \"4 bedrooms\"'    # THIS IS CRITICAL FOR THE LLM TO IDENTIFY THE CATEGORY SINCE NO MEMORY -> NO LINK BETWEEN QUESTION AND ANSWER (answer must make the obvious link with the subject of the question)\n",
    "    print(YELLOW + ai_response + ENDC)\n",
    "    customer_input = input(ai_response)\n",
    "    print(GREEN + customer_input + ENDC)\n",
    "    user_preferences, ask_for = filter_response(customer_input, user_preferences)\n",
    "else:\n",
    "    print('Thank you ! Everything has been gathered, moving now to next screening phase')\n",
    "\n",
    "# final question to collect the gist of the ideal property\n",
    "print(YELLOW + 'Besides these criteria, describe the property you would love to buy ?' + ENDC)\n",
    "customer_input = input('Besides these criteria, describe the property you would love to buy ?')\n",
    "print(GREEN + customer_input + ENDC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 690,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PreferenceDetails(price=2500000, bedrooms=4, bathrooms=3, size=180)"
      ]
     },
     "execution_count": 690,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_preferences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 691,
   "id": "c037e835",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AIMessage(content='', additional_kwargs={'function_call': {'name': 'query_db', 'arguments': '{\"query\":\"quiet neighborhood, proximity to city\",\"n_results\":3}'}})"
      ]
     },
     "execution_count": 691,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# TRIGGER FUNCTION CALLING PARAMETERS\n",
    "ai_response = chat_llm.predict_messages([HumanMessage(content=customer_input + ' Characteristics: ' + str(user_preferences.dict()))], functions=functions)\n",
    "# similar to response = chat_llm.invoke(\"i am looking for a house with 3 bedrooms and a view on the lake?\", functions=functions)\n",
    "ai_response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 692,
   "id": "51c6b225",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "query_db\n",
      "{'query': 'quiet neighborhood, proximity to city', 'n_results': 3}\n",
      "{'ids': [['5', '14', '15']], 'distances': [[0.3490514690109264, 0.36467161012339894, 0.36591028592542746]], 'metadatas': [[{'bathrooms': 2, 'bedrooms': 3, 'house size': 180, 'neighborhood': 'Downtown District', 'price': 2000000}, {'bathrooms': 2, 'bedrooms': 3, 'house size': 170, 'neighborhood': 'Cedar Valley', 'price': 1600000}, {'bathrooms': 3, 'bedrooms': 4, 'house size': 260, 'neighborhood': 'City Center', 'price': 2500000}]], 'embeddings': None, 'documents': [['Urban Chic in Downtown District: Embrace urban chic living in this stylish 3-bedroom, 2-bathroom home in the heart of the Downtown District. With modern design, high-end appliances, and a rooftop patio for entertaining, this property is perfect for those who love city living. Neighborhood: The Downtown District is a bustling urban center with a vibrant nightlife, trendy restaurants, and cultural attractions. Experience the excitement of city life right outside your doorstep.', 'Tranquil Retreat in Cedar Valley: Find peace and tranquility in this 3-bedroom, 2-bathroom home in Cedar Valley. Surrounded by cedar trees, with a wrap-around porch and a meditation garden, this property offers a serene retreat in a natural setting. Neighborhood: Cedar Valley is a peaceful neighborhood with hiking trails, nature reserves, and a strong sense of community. Enjoy the beauty of the outdoors and a slower pace of life in this idyllic enclave.', 'Urban Oasis in City Center: Discover an urban oasis in the heart of the City Center. This 4-bedroom, 3-bathroom home features modern design, a rooftop terrace with city views, and a home office, offering a perfect blend of style and functionality. Neighborhood: City Center is a vibrant urban hub with shopping centers, restaurants, and cultural attractions. Experience the convenience and excitement of city living in this bustling neighborhood.']]}\n",
      "\u001b[33mDistance 0.349 : Urban Chic in Downtown District\u001b[0m\n",
      "- neighborhood              : Downtown District\n",
      "- price                     : 2000000\n",
      "- bedrooms                  : 3\n",
      "- bathrooms                 : 2\n",
      "- house size                : 180\n",
      "- description               : Embrace urban chic living in this stylish 3-bedroom, 2-bathroom home in the heart of the Downtown District. With modern design, high-end appliances, and a rooftop patio for entertaining, this property is perfect for those who love city living.\n",
      "- neighborhood description  : The Downtown District is a bustling urban center with a vibrant nightlife, trendy restaurants, and cultural attractions. Experience the excitement of city life right outside your doorstep.\n",
      "-------------------------------------------------\n",
      "\u001b[33mDistance 0.365 : Tranquil Retreat in Cedar Valley\u001b[0m\n",
      "- neighborhood              : Cedar Valley\n",
      "- price                     : 1600000\n",
      "- bedrooms                  : 3\n",
      "- bathrooms                 : 2\n",
      "- house size                : 170\n",
      "- description               : Find peace and tranquility in this 3-bedroom, 2-bathroom home in Cedar Valley. Surrounded by cedar trees, with a wrap-around porch and a meditation garden, this property offers a serene retreat in a natural setting.\n",
      "- neighborhood description  : Cedar Valley is a peaceful neighborhood with hiking trails, nature reserves, and a strong sense of community. Enjoy the beauty of the outdoors and a slower pace of life in this idyllic enclave.\n",
      "-------------------------------------------------\n",
      "\u001b[33mDistance 0.366 : Urban Oasis in City Center\u001b[0m\n",
      "- neighborhood              : City Center\n",
      "- price                     : 2500000\n",
      "- bedrooms                  : 4\n",
      "- bathrooms                 : 3\n",
      "- house size                : 260\n",
      "- description               : Discover an urban oasis in the heart of the City Center. This 4-bedroom, 3-bathroom home features modern design, a rooftop terrace with city views, and a home office, offering a perfect blend of style and functionality.\n",
      "- neighborhood description  : City Center is a vibrant urban hub with shopping centers, restaurants, and cultural attractions. Experience the convenience and excitement of city living in this bustling neighborhood.\n",
      "-------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>listings_headline</th>\n",
       "      <th>listings_neighborhood</th>\n",
       "      <th>listings_price</th>\n",
       "      <th>listings_bedrooms</th>\n",
       "      <th>listings_bathrooms</th>\n",
       "      <th>listings_house_size</th>\n",
       "      <th>listings_description</th>\n",
       "      <th>listings_neighborhood_description</th>\n",
       "      <th>full_description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Urban Chic in Downtown District</td>\n",
       "      <td>Downtown District</td>\n",
       "      <td>2000000</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>180</td>\n",
       "      <td>Embrace urban chic living in this stylish 3-be...</td>\n",
       "      <td>The Downtown District is a bustling urban cent...</td>\n",
       "      <td>Urban Chic in Downtown District: Embrace urban...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Tranquil Retreat in Cedar Valley</td>\n",
       "      <td>Cedar Valley</td>\n",
       "      <td>1600000</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>170</td>\n",
       "      <td>Find peace and tranquility in this 3-bedroom, ...</td>\n",
       "      <td>Cedar Valley is a peaceful neighborhood with h...</td>\n",
       "      <td>Tranquil Retreat in Cedar Valley: Find peace a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Urban Oasis in City Center</td>\n",
       "      <td>City Center</td>\n",
       "      <td>2500000</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>260</td>\n",
       "      <td>Discover an urban oasis in the heart of the Ci...</td>\n",
       "      <td>City Center is a vibrant urban hub with shoppi...</td>\n",
       "      <td>Urban Oasis in City Center: Discover an urban ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   listings_headline listings_neighborhood  listings_price  \\\n",
       "5    Urban Chic in Downtown District     Downtown District         2000000   \n",
       "14  Tranquil Retreat in Cedar Valley          Cedar Valley         1600000   \n",
       "15        Urban Oasis in City Center           City Center         2500000   \n",
       "\n",
       "    listings_bedrooms  listings_bathrooms  listings_house_size  \\\n",
       "5                   3                   2                  180   \n",
       "14                  3                   2                  170   \n",
       "15                  4                   3                  260   \n",
       "\n",
       "                                 listings_description  \\\n",
       "5   Embrace urban chic living in this stylish 3-be...   \n",
       "14  Find peace and tranquility in this 3-bedroom, ...   \n",
       "15  Discover an urban oasis in the heart of the Ci...   \n",
       "\n",
       "                    listings_neighborhood_description  \\\n",
       "5   The Downtown District is a bustling urban cent...   \n",
       "14  Cedar Valley is a peaceful neighborhood with h...   \n",
       "15  City Center is a vibrant urban hub with shoppi...   \n",
       "\n",
       "                                     full_description  \n",
       "5   Urban Chic in Downtown District: Embrace urban...  \n",
       "14  Tranquil Retreat in Cedar Valley: Find peace a...  \n",
       "15  Urban Oasis in City Center: Discover an urban ...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mDistance 0.349 : Urban Chic in Downtown District\u001b[0m\n",
      "- neighborhood              : Downtown District\n",
      "- price                     : 2000000\n",
      "- bedrooms                  : 3\n",
      "- bathrooms                 : 2\n",
      "- house size                : 180\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "<span style=\"color:cyan\">augmented description : \n",
       "Looking for the perfect blend of city living and peaceful surroundings? Look no further than this stunning 3-bedroom, 2-bathroom home in the heart of the Downtown District. With its modern design, high-end appliances, and rooftop patio for entertaining, this property is the epitome of urban chic. And with its location in a quiet neighborhood just steps away from the bustling city center, you can have the best of both worlds. Don't miss out on the opportunity to experience the excitement of city life while still enjoying a tranquil living environment.</span>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------\n",
      "\u001b[33mDistance 0.365 : Tranquil Retreat in Cedar Valley\u001b[0m\n",
      "- neighborhood              : Cedar Valley\n",
      "- price                     : 1600000\n",
      "- bedrooms                  : 3\n",
      "- bathrooms                 : 2\n",
      "- house size                : 170\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "<span style=\"color:cyan\">augmented description : \n",
       "Welcome to your tranquil retreat in Cedar Valley! This stunning 3-bedroom, 2-bathroom home is nestled among the beautiful cedar trees, providing a peaceful and serene atmosphere. Step onto the wrap-around porch and take in the natural surroundings, or find your inner peace in the meditation garden. Located in the quiet neighborhood of Cedar Valley, you'll have access to hiking trails, nature reserves, and a strong sense of community. Enjoy the best of both worlds with a peaceful escape from the city, while still being in close proximity. Don't miss out on this idyllic enclave - schedule a viewing today!</span>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------\n",
      "\u001b[33mDistance 0.366 : Urban Oasis in City Center\u001b[0m\n",
      "- neighborhood              : City Center\n",
      "- price                     : 2500000\n",
      "- bedrooms                  : 4\n",
      "- bathrooms                 : 3\n",
      "- house size                : 260\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "<span style=\"color:cyan\">augmented description : \n",
       "Welcome to your dream home in the City Center! This stunning 4-bedroom, 3-bathroom oasis offers the perfect balance of tranquility and city living. With a modern design and a rooftop terrace boasting breathtaking city views, you'll never want to leave. Need to work from home? No problem, this home also features a convenient home office. And with its prime location in the bustling City Center, you'll have easy access to shopping, dining, and cultural attractions. Say goodbye to noisy neighborhoods and hello to your peaceful urban retreat.</span>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "Based on your preferences, here are the top 3 property matches:\n",
       "\n",
       "1. Urban Chic in Downtown District\n",
       "   - Neighborhood: Downtown District\n",
       "   - Price: $2,000,000\n",
       "   - Bedrooms: 3\n",
       "   - Bathrooms: 2\n",
       "   - House Size: 180 sq. ft\n",
       "   - Description: Looking for the perfect blend of city living and peaceful surroundings? Look no further than this stunning 3-bedroom, 2-bathroom home in the heart of the Downtown District. With its modern design, high-end appliances, and rooftop patio for entertaining, this property is the epitome of urban chic. And with its location in a quiet neighborhood just steps away from the bustling city center, you can have the best of both worlds. Don't miss out on the opportunity to experience the excitement of city life while still enjoying a tranquil living environment.\n",
       "\n",
       "2. Tranquil Retreat in Cedar Valley\n",
       "   - Neighborhood: Cedar Valley\n",
       "   - Price: $1,600,000\n",
       "   - Bedrooms: 3\n",
       "   - Bathrooms: 2\n",
       "   - House Size: 170 sq. ft\n",
       "   - Description: Welcome to your tranquil retreat in Cedar Valley! This stunning 3-bedroom, 2-bathroom home is nestled among the beautiful cedar trees, providing a peaceful and serene atmosphere. Step onto the wrap-around porch and take in the natural surroundings, or find your inner peace in the meditation garden. Located in the quiet neighborhood of Cedar Valley, you'll have access to hiking trails, nature reserves, and a strong sense of community. Enjoy the best of both worlds with a peaceful escape from the city, while still being in close proximity. Don't miss out on this idyllic enclave - schedule a viewing today!\n",
       "\n",
       "3. Urban Oasis in City Center\n",
       "   - Neighborhood: City Center\n",
       "   - Price: $2,500,000\n",
       "   - Bedrooms: 4\n",
       "   - Bathrooms: 3\n",
       "   - House Size: 260 sq. ft\n",
       "   - Description: Welcome to your dream home in the City Center! This stunning 4-bedroom, 3-bathroom oasis offers the perfect balance of tranquility and city living. With a modern design and a rooftop terrace boasting breathtaking city views, you'll never want to leave. Need to work from home? No problem, this home also features a convenient home office. And with its prime location in the bustling City Center, you'll have easy access to shopping, dining, and cultural attractions. Say goodbye to noisy neighborhoods and hello to your peaceful urban retreat."
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if ai_response.content=='':\n",
    "    # EXECUTE FUNCTION CALLING\n",
    "    tool_name = ai_response.additional_kwargs['function_call']['name']\n",
    "    tool = [tools[i] for i, tool in enumerate(tools) if tool.name==tool_name][0]\n",
    "    _args = json.loads(ai_response.additional_kwargs['function_call']['arguments'])\n",
    "\n",
    "    print(tool.name)\n",
    "    print(_args)\n",
    "    extrated_properties = tool(_args)\n",
    "    print(extrated_properties)\n",
    "    # narrow down the list using threshold parameter\n",
    "    matches_df = display_matches(extrated_properties,0.37)\n",
    "    display(matches_df)\n",
    "\n",
    "    # generate augmented personalized descriptions\n",
    "    augmented_df = augment_properties(extrated_properties, customer_input, matches_df)\n",
    "    # Generate final LLM response in natural language\n",
    "    final_output = chat_llm.predict_messages([ \n",
    "            HumanMessage(content=customer_input),\n",
    "            ai_response,\n",
    "            FunctionMessage(name=tool.name, content=str(extrated_properties)),\n",
    "            AIMessage(content=str((augmented_df\n",
    "                                .drop(['listings_description','listings_neighborhood_description','full_description'], axis=1,errors='ignore')  # we keep the augmented description and the metadata characteristics\n",
    "                                .to_dict(orient='dict')\n",
    "                                )\n",
    "                                ))\n",
    "                                ], \n",
    "                                functions=functions\n",
    "                            )\n",
    "    # Print final recommendation in natural language\n",
    "    display(Markdown(final_output.content))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ee04a50",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fab62991",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57c9ce14",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5099628e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b693ad56",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "918c1c34",
   "metadata": {},
   "source": [
    "# OUT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 511,
   "id": "3d94531f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "Great! A house with 3 bedrooms and a view of the lake sounds wonderful. Are there any other specific features or amenities you are looking for in the property? For example, do you prefer a modern or traditional style, a certain neighborhood, or any specific budget in mind?"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "'''system_template=\"You are a real estate agent. You engage conversation with the customers to understand their needs and provide them with the best possible property. Your task is to collect the necessary selection criteria of the customer.\"\n",
    "system_message_prompt = SystemMessagePromptTemplate.from_template(system_template)\n",
    "ai_template_1 = \"Hi! I am here to help you find the best property for you. Could you please tell me about the ideal characteristics of the property you are lookiging for?\"\n",
    "ai_message_1= AIMessagePromptTemplate.from_template(ai_template_1)\n",
    "human_template=\"{customer_input}\" \n",
    "human_message_prompt = HumanMessagePromptTemplate.from_template(human_template)\n",
    "ai_template_2=\"{response}\"\n",
    "ai_message_2 = AIMessagePromptTemplate.from_template(ai_template_2)\n",
    "chat_prompt = ChatPromptTemplate.from_messages([system_message_prompt,ai_message_1, human_message_prompt,ai_message_2])\n",
    "llm_chain = LLMChain(prompt=chat_prompt, llm=chat_llm)\n",
    "customer_input=\"i am looking for a house with 3 bedrooms and a view on the lake. What do you propose?\"\n",
    "response1 = llm_chain.run(response=\"\", customer_input=customer_input, functions=functions)\n",
    "display(Markdown(response1))'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfe242eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\"\"\"\n",
    "\n",
    "prompt = '''You are a helpful telcom sales assistant. In order to help someone to the right plan you need the city a person is based on wether the person is look for an individual plan or a family plan. \n",
    "\n",
    "For an individual plan- just ask the age of the person. \n",
    "\n",
    "If It is a family plan - we need to know how many people are on it and what there ages are for each person. \n",
    "The max for a family plan is set to 5. \n",
    "In order to provide a quote - you will need to have the ages of all people on the plan.\n",
    "\n",
    "When people provide only the city name, please infer the State and then confirm with the person. Like when they say 'Atlanta' - you ask something like 'So you are in Atlanta, Georgia right?\n",
    "\n",
    "Once you have all the information (Individual or family plan, number of people and age for EACH person, city and state) you can call the plan function that will return the price. '''\n",
    "functions = [\n",
    "\n",
    "{\n",
    "  \"name\": \"getplanprice\",\n",
    "  \"parameters\": {\n",
    "    \"type\": \"object\",\n",
    "    \"properties\": {\n",
    "      \"plan\": {\n",
    "        \"type\": \"string\",\n",
    "        \"Description\": \"Individual or Family\"\n",
    "      },\n",
    "      \"people\": {\n",
    "        \"type\": \"integer\",\n",
    "        \"description\": \"The number of people on the plan\"\n",
    "      },\n",
    "      \"ages\": {\n",
    "        \"type\": \"string\",\n",
    "        \"description\": \"Commma separates list of the ages of all plan participants\"\n",
    "      }\n",
    "    },\n",
    "    \"required\": [\n",
    "      \"plan\",\n",
    "      \"people\",\n",
    "      \"ages\"\n",
    "    ]\n",
    "  },\n",
    "  \"description\": \"Needs the city/state, the type of plan, the number of peole on the plan and the ages of the people\"\n",
    "}\n",
    "\n",
    "\n",
    "]\n",
    "\n",
    "- we use a set of defined Q & A\n",
    "personal_questions = [  \"Which movie genre you like the most?\", \n",
    "                        \"What is your favorite color?\", \n",
    "                        \"What is your favorite movie?\", \n",
    "                        \"Pick one - dogs, cats or hamsters?\",\n",
    "                        \"What is your favorite food?\",\n",
    "                        \"What is your favorite drink?\" ]\n",
    "\n",
    "#personal_answers = [ ] \n",
    "#for question in personal_questions:\n",
    "#    answer = input(question)\n",
    "#    personal_answers.append(answer)\n",
    "    \n",
    "personal_answers = ['thriller', 'blue', 'inception', 'dogs', 'fish tacos', 'cold beer']\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9812ca9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "987c50f5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2ff0dfb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df8d6a72",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ffe34211",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}