Added Indexing and One Time Database Loading

initializer/database.py

@@ -7,5 +7,10 @@ def Initializer_Database():
     cursor.execute('''CREATE TABLE IF NOT EXISTS information
                     (name TEXT, address TEXT, text TEXT)''')
 
+    cursor.execute('''CREATE INDEX IF NOT EXISTS idx_name ON information (name)''')
+    cursor.execute('''CREATE INDEX IF NOT EXISTS idx_address ON information (address)''')
+
     conn.commit()
     conn.close()
+
+    return conn

initializer/loader.py

@@ -0,0 +1,5 @@
+import sqlite3
+
+def database_loader():
+    conn = sqlite3.connect('./database/search-index.db')
+    return conn

main.py

@@ -1,12 +1,15 @@
 import streamlit as st
 import time
 
+from initializer.loader import database_loader
 from manager.insert import insert_data
 from search.index import Search_Data
 from initializer.database import Initializer_Database
 
 Initializer_Database()
 
+conn = database_loader()
+
 st.title('MonoSearch')
 
 st.session_state.setdefault('add_state', False)
@@ -25,7 +28,7 @@
         submitted2 = st.form_submit_button('Add')
 
     if keyword and submitted1:
-        Search_Data(keyword)
+        Search_Data(conn, keyword)
 
     if submitted2 and AddForm == True:
         name = st.text_input('Enter website name:')
@@ -34,7 +37,7 @@
         if name and address:
             with st.spinner('Checking the given information...'):
                 time.sleep(1)
-                insert_data(name, address)
+                insert_data(conn, name, address)
                 st.session_state.add_state = False
     elif submitted2 and not AddForm:
         st.session_state.add_state = True

manager/insert.py

@@ -6,8 +6,8 @@
 
 GOOGLE_SAFE_BROWSING_API_KEY = 'API_KEY'
 
-def content_exists(text_content, address):
-    with sqlite3.connect('./database/search-index.db') as conn:
+def content_exists(conn, text_content, address):
+    with conn:
         cursor = conn.cursor()
         if text_content:
             cursor.execute('''SELECT COUNT(*) FROM information WHERE text = ?''', (text_content,))
@@ -41,7 +41,7 @@ def is_content_safe(text_content):
             return False
     return True
 
-def insert_data(name, address):
+def insert_data(conn, name, address):
     try:
         response = requests.get(address)
         response.raise_for_status()
@@ -51,15 +51,15 @@ def insert_data(name, address):
         st.error("Error accessing or parsing the website:", e)
         return
 
-    if content_exists(text_content, address):
+    if content_exists(conn, text_content, address):
         st.warning("Content already exists in the database.")
         return
 
     if not is_content_safe(address):
         st.warning("Unsafe content detected. Not inserting into the database.")
         return
 
-    with sqlite3.connect('./database/search-index.db') as conn:
+    with conn:
         cursor = conn.cursor()
         try:
             cursor.execute('''INSERT INTO information (name, address, text)

requirements.txt

@@ -1,3 +1,3 @@
 streamlit
 requests
-bs4
+bs4

search/index.py

@@ -8,11 +8,12 @@ def summary(text):
     else:
         return ' '.join(words[:30]) + ' ...'
 
-def Search_Data(keyword):
-    conn = sqlite3.connect('./database/search-index.db')
+def Search_Data(conn, keyword):
     cursor = conn.cursor()
+
     cursor.execute('''SELECT * FROM information
                       WHERE name LIKE ? OR address LIKE ?''', ('%' + keyword + '%', '%' + keyword + '%'))
+
     rows = cursor.fetchall()
     conn.close()