notebook_openai.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import openai\n",
    "import sys\n",
    "\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "\n",
    "from glob import glob\n",
    "from os.path import join\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "from langchain.document_loaders import PyPDFLoader\n",
    "\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
    "from langchain.vectorstores import Chroma\n",
    "\n",
    "from langchain.llms import OpenAI\n",
    "\n",
    "from langchain.retrievers import ContextualCompressionRetriever\n",
    "from langchain.retrievers.document_compressors import LLMChainExtractor\n",
    "\n",
    "from langchain.prompts import PromptTemplate\n",
    "from langchain.chains import RetrievalQA, ConversationalRetrievalChain\n",
    "\n",
    "from langchain.memory import ConversationBufferMemory\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = load_dotenv(find_dotenv()) # read local .env file\n",
    "\n",
    "openai.api_key  = os.environ['OPENAI_API_KEY']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating ChromaDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/34 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\35-GHz Barium Hexaferrite or PDMS Composite-Based Millimeter-Wave Circulators for 5G Applications.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|▎         | 1/34 [00:01<00:33,  1.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\A comprehensive survey on machine learning for networking - Evolution, applications and research opportunities.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|▌         | 2/34 [00:06<01:56,  3.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\A gated dilated causal convolution based encoder-decoder for network traffic forecasting.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|▉         | 3/34 [00:07<01:13,  2.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\A network traffic forecasting method based on SA optimized ARIMA-BP neural network.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|█▏        | 4/34 [00:11<01:30,  3.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\A Self-Adaptive Deep Learning-Based System for Anomaly Detection in 5G Networks.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▍        | 5/34 [00:14<01:23,  2.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\A Survey on Big Data for Network Traffic Monitoring and Analysis.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 6/34 [00:16<01:17,  2.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Analyzing and modeling spatio-temporal dependence of cellular traffic at city scale.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|██        | 7/34 [00:21<01:37,  3.62s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Application of Machine Learning in Wireless Networks - Key Techniques and Open Issues.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▎       | 8/34 [00:25<01:36,  3.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Backhauling 5G Small Cells - A Radio Resource Management Perspective.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██▋       | 9/34 [00:26<01:07,  2.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Beyond Moran’s I - Testing for spatial dependence based on the spatial autoregressive model.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|██▉       | 10/34 [00:31<01:21,  3.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Big data-driven optimization for mobile networks toward 5G.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 11/34 [00:32<01:04,  2.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Characterizing the spatio-temporal inhomogeneity of mobile traffic in large-scale cellular data networks.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 13/34 [00:33<00:33,  1.60s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Clustering to Enhance Network Traffic Forecasting.pdf\n",
      "data\\Deep learning in mobile and wireless networking - A survey.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████      | 14/34 [00:39<00:59,  2.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Deploying Virtual Network Functions With Non-Uniform Models in Tree-Structured Networks.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▍     | 15/34 [00:43<01:01,  3.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Design of mm-Wave Slow-Wave-Coupled Coplanar Waveguides.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 16/34 [00:49<01:09,  3.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Exploring Network-Wide Flow Data With Flowyager.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 17/34 [01:30<04:19, 15.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Generative-Adversarial-Network-Based wireless channel modeling - Challenges and opportunities.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 18/34 [01:31<02:52, 10.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Hierarchical, virtualised and distributed intelligence 5G architecture for low-latency and secure applications.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|█████▌    | 19/34 [01:31<01:55,  7.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Improving traffic forecasting for 5G core network scalability - A machine learning approach.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59%|█████▉    | 20/34 [01:33<01:20,  5.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Long-term mobile traffic forecasting using deep spatio-temporal neural networks.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▏   | 21/34 [01:44<01:38,  7.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Massive MIMO CSI Feedback Based on Generative Adversarial Network.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▍   | 22/34 [01:45<01:05,  5.46s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Orchestrating Virtualized Network Functions.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████▊   | 23/34 [01:49<00:55,  5.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Representational power of Restricted Boltzmann Machines and Deep Belief Networks.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71%|███████   | 24/34 [01:49<00:36,  3.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Resource Allocation in NFV- A Comprehensive Survey.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████▎  | 25/34 [01:55<00:38,  4.31s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Resource Sharing Efficiency in Network_Slicing.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████▋  | 26/34 [02:06<00:49,  6.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Spatial modeling of the traffic density in cellular networks.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|████████▏ | 28/34 [02:06<00:18,  3.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Spatial Traffic Distribution In Cellular Networks.pdf\n",
      "data\\Spatio-temporal analysis and prediction of cellular traffic in metropolis.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▌ | 29/34 [02:08<00:14,  2.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Spatiotemporal modeling and prediction in cellular networks - A big data enabled deep learning approach.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 30/34 [02:09<00:08,  2.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\TANGO - Traffic-Aware Network Planning and Green Operation.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 31/34 [02:09<00:04,  1.62s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Time4 - Time for SDN.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|█████████▍| 32/34 [02:13<00:04,  2.30s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Towards Supporting Intelligence in 5G-6G Core.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████▋| 33/34 [02:14<00:01,  1.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\\Understanding Mobile Traffic Patterns of Large Scale Cellular Towers in Urban Environment.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 34/34 [02:18<00:00,  4.09s/it]\n"
     ]
    }
   ],
   "source": [
    "files_dir = 'data'\n",
    "\n",
    "# Use a função glob para listar todos os arquivos PDF no diretório\n",
    "pdf_files = glob(f'{files_dir}/*.pdf')\n",
    "\n",
    "docs=[]\n",
    "\n",
    "for arquivo in tqdm(pdf_files):\n",
    "    print(arquivo)\n",
    "\n",
    "    loader = PyPDFLoader(arquivo)\n",
    "    docs.extend(loader.load())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size = 1500,\n",
    "    chunk_overlap = 150\n",
    ")\n",
    "\n",
    "splits = text_splitter.split_documents(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\PatrickLuizdeAraujo\\anaconda3\\envs\\watsonxai\\Lib\\site-packages\\langchain_core\\_api\\deprecation.py:117: LangChainDeprecationWarning: The class `langchain_community.embeddings.openai.OpenAIEmbeddings` was deprecated in langchain-community 0.1.0 and will be removed in 0.2.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import OpenAIEmbeddings`.\n",
      "  warn_deprecated(\n"
     ]
    }
   ],
   "source": [
    "#!rm -Rf /Users/patrick/Documents/Pessoal/paper-rag/chroma/\n",
    "\n",
    "embedding = OpenAIEmbeddings()\n",
    "persist_directory = 'chroma_full'\n",
    "\n",
    "vectordb = Chroma.from_documents(\n",
    "    documents=splits,\n",
    "    embedding=embedding,\n",
    "    persist_directory=persist_directory\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\PatrickLuizdeAraujo\\anaconda3\\envs\\watsonxai\\Lib\\site-packages\\langchain_core\\_api\\deprecation.py:117: LangChainDeprecationWarning: The class `langchain_community.llms.openai.OpenAI` was deprecated in langchain-community 0.0.10 and will be removed in 0.2.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import OpenAI`.\n",
      "  warn_deprecated(\n"
     ]
    }
   ],
   "source": [
    "llm = OpenAI(temperature=0)  # Model used"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Template"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build prompt\n",
    "template = \"\"\"Use the following pieces of context to answer the question at the end. If you can't make a answer with context, just say that you don't know, don't try to make up an answer. Give the document name from where the information is extracted\n",
    "{context}\n",
    "Question: {question}\n",
    "Helpful Answer:\"\"\"\n",
    "\n",
    "QA_CHAIN_PROMPT = PromptTemplate.from_template(template)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compression retrieval method configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Wrap our vectorstore\n",
    "compressor = LLMChainExtractor.from_llm(llm)  # uses an LLMChain to extract from each document only the statements that are relevant to the query.\n",
    "\n",
    "compression_retriever = ContextualCompressionRetriever(\n",
    "    base_compressor=compressor,\n",
    "    base_retriever=vectordb.as_retriever(search_type = \"mmr\")  # Retriever method using MMR retrieval to ensure variability in the answer\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chain assembling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run chain\n",
    "qa_chain = RetrievalQA.from_chain_type(\n",
    "    llm,\n",
    "    retriever=compression_retriever,\n",
    "    return_source_documents=True,\n",
    "    chain_type_kwargs={\"prompt\": QA_CHAIN_PROMPT},\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\PatrickLuizdeAraujo\\anaconda3\\envs\\watsonxai\\Lib\\site-packages\\langchain_core\\_api\\deprecation.py:117: LangChainDeprecationWarning: The function `__call__` was deprecated in LangChain 0.1.0 and will be removed in 0.2.0. Use invoke instead.\n",
      "  warn_deprecated(\n",
      "c:\\Users\\PatrickLuizdeAraujo\\anaconda3\\envs\\watsonxai\\Lib\\site-packages\\langchain\\chains\\llm.py:316: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
      "  warnings.warn(\n",
      "c:\\Users\\PatrickLuizdeAraujo\\anaconda3\\envs\\watsonxai\\Lib\\site-packages\\langchain\\chains\\llm.py:316: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
      "  warnings.warn(\n",
      "c:\\Users\\PatrickLuizdeAraujo\\anaconda3\\envs\\watsonxai\\Lib\\site-packages\\langchain\\chains\\llm.py:316: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
      "  warnings.warn(\n",
      "c:\\Users\\PatrickLuizdeAraujo\\anaconda3\\envs\\watsonxai\\Lib\\site-packages\\langchain\\chains\\llm.py:316: UserWarning: The predict_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " The Nnwdaf_AnalyticsSubscriptionservice is a service provided by the NWDAF (Network Data Analytics Function) that notifies the NF (Network Function) consumer instance of all analytics subscribed to the specific NWDAF service. This information is extracted from the document \"NWDAF Services and Interfaces\" which describes the different services provided by the NWDAF.\n",
      "\n",
      "\n",
      "Source:  data\\Towards Supporting Intelligence in 5G-6G Core.pdf\n",
      "Page:  1\n",
      "\n",
      "\n",
      "Source:  data\\Resource Sharing Efficiency in Network_Slicing.pdf\n",
      "Page:  5\n",
      "\n",
      "\n",
      "Source:  data\\Improving traffic forecasting for 5G core network scalability - A machine learning approach.pdf\n",
      "Page:  5\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "question = \"What is Nnwdaf_AnalyticsSubscriptionservice?\"\n",
    "result = qa_chain({\"query\": question})\n",
    "\n",
    "print(result[\"result\"])\n",
    "print('\\n')\n",
    "\n",
    "for source in result['source_documents']:\n",
    "    print ('Source: ', source.metadata['source'].split('/')[-1])\n",
    "    print('Page: ', source.metadata['page'])\n",
    "    print('\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "langflow",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}