Document query

Author: sudheesh4

Diff for: docquery.py

@@ -0,0 +1,101 @@
+from langchain.embeddings import SentenceTransformerEmbeddings #HuggingFaceInstructEmbeddings
+from langchain.vectorstores import FAISS
+import os 
+import copy
+import pprint
+#import google.generativeai as palm
+from langchain.llms import GooglePalm
+from langchain import PromptTemplate
+from langchain.chains import RetrievalQA
+
+PALM_API="AIzaSyAIzDH7NVopxUvOL8PAqBnKZqdmAoXeS28"
+#palm.configure(api_key=PALM_API)
+
+
+def getmodel():
+    "test"
+    PALM_API="AIzaSyAIzDH7NVopxUvOL8PAqBnKZqdmAoXeS28"
+    embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
+    db = FAISS.load_local("faiss", embeddings)
+    retriever = db.as_retriever(search_kwargs={'k': 10})
+    #prompt=getprompt()
+    llm=GooglePalm(google_api_key=PALM_API,temperature=0.00003,max_output_tokens=512)
+    qa_llm = RetrievalQA.from_chain_type(llm=llm,
+                                          chain_type='refine',
+                                          retriever=retriever,
+                                          return_source_documents=True,
+                                          #chain_type_kwargs={'prompt': prompt},
+                                        verbose=True)
+    return qa_llm
+
+def getprompt():
+    template = """Use the information to elaborate in points about the user's query.
+    If user mentions something not in the 'Context', just answer that you don't know.
+    If you don't know the answer, just say that you don't know, don't try to make up an answer.
+    
+    Context: {context}
+    
+    Query: {question}
+    
+    Only return the helpful answer below and nothing else.
+    
+    Helpful answer:
+    """
+    prompt = PromptTemplate(
+        template=template,
+        input_variables=['context', 'question'])
+    return prompt
+
+def parseresult(result):
+    
+    PARSED=copy.deepcopy(result)
+    docs=PARSED['source_documents']
+    sourcepage=[]
+    for d in docs:
+        sourcepage.append(d.metadata['page'])
+    PARSED['source_pages']=copy.deepcopy(sourcepage)
+    del sourcepage,result
+    return PARSED
+
+def getsources(result):
+    sources=[]
+    for s in result['source_documents']:
+        sources.append(f"{s.metadata}")
+    return sources
+
+def EXTRACT():
+    print(f"{'>>>'*17} QUERY DOCS{'<<<'*17}")
+    try:
+        llm=getmodel()
+    except:
+        print("CANNOT LOAD MODEL OR DATABASE")
+        print(f"{'###'*40}")
+        return
+    while True:
+        print(f"{'###'*40}")
+        prompt=input("(To stop querying enter exit) \n Query  : ")
+
+        if prompt:
+            if prompt.find('exit')==0:
+                return
+            else:
+                pass
+
+            try:
+                result=parseresult(llm(prompt))
+                sources=getsources(result)
+                result=result["result"]
+            except:
+                result='Error ocurred!'
+                sources=[]
+            print(f"{'!!!'*40}")
+            print(f"QUERY: {prompt}")
+            print(f"{'###'*40}")
+            print("RESULT:")
+            #print(f"{'###'*40}")
+            print(result)
+            print(f"{'$$$'*40}")
+            print("SOURCES:")
+            #print(f"{'$$$'*40}")
+            print(sources)
+            print(f"{'>>>'*40}")

Diff for: ingest.py

@@ -0,0 +1,44 @@
+from langchain.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter 
+from langchain.embeddings import SentenceTransformerEmbeddings #HuggingFaceInstructEmbeddings
+from langchain.vectorstores import FAISS
+
+import os
+def getdoctext(dir):
+    os.chdir(dir)
+    files=os.listdir()
+    texts=[]
+    text_splitter = RecursiveCharacterTextSplitter(separators=["\n\n","\n","."," "],chunk_size=2000, chunk_overlap=100)
+    for file in files:
+        if ".pdf" in file:
+            print(file)
+            texts=texts + text_splitter.split_documents(PyPDFLoader(file).load())
+    os.chdir("..")
+    return texts
+
+def pageextract(texts):
+    PAGES=[]
+    id=0
+    while id<len(texts):
+        if (texts[id].metadata['page']-7)==len(PAGES):
+            temp=''
+            while id<len(texts) and (texts[id].metadata['page']-7)==len(PAGES) :
+                temp=temp+texts[id].page_content
+                id=id+1
+            PAGES.append(copy.deepcopy(texts[0]))
+            PAGES[-1].page_content=temp
+            PAGES[-1].metadata['page']=1*len(PAGES)
+    
+    del PAGES
+
+def makedb(chunks,embeddings):
+    
+    db = FAISS.from_documents(chunks, embeddings)
+    db.save_local("faiss")
+    del db
+
+    
+def INGESTER():
+    chunks=getdoctext("data")
+    embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
+    makedb(chunks,embeddings)

Diff for: streamlit_app.py

@@ -0,0 +1,36 @@
+from langchain.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter 
+from langchain.embeddings import SentenceTransformerEmbeddings #HuggingFaceInstructEmbeddings
+from langchain.vectorstores import FAISS
+import os 
+import copy
+import time
+import pprint
+#import google.generativeai as palm
+from langchain.llms import GooglePalm
+from langchain import PromptTemplate
+from langchain.chains import RetrievalQA
+#import streamlit as st
+import os
+import subprocess
+import sentence_transformers
+import nltk
+
+while True:
+    choice=input(f"{'>>'*10} \n Choose : 0-> Ingest ; 1->Query ; 2->Exit \n : ")
+
+    if choice=='0':
+        from ingest import INGESTER
+        print("INGESTING!")
+        INGESTER()
+        print("Ingested")
+    elif choice=='1':
+        from docquery import EXTRACT
+        print("QUERYING!")
+        EXTRACT()
+    elif choice=='2':
+        print("Exiting!")
+        time.sleep(2)
+        break
+    else:
+        print("Invalid choice!")