Add local file knowledge base

Author: blmdxiao

crawler_module/__init__.py

@@ -0,0 +1,9 @@
+LLM_NAME="OpenAI"
+OPENAI_API_KEY="xxxx"
+GPT_MODEL_NAME="gpt-3.5-turbo"
+MIN_RELEVANCE_SCORE=0.3
+BOT_TOPIC="xxxx"
+URL_PREFIX="http://127.0.0.1:7000/"
+USE_PREPROCESS_QUERY=0
+USE_RERANKING=1
+USE_DEBUG=0

crawler_module/document_embedding.py

@@ -0,0 +1,9 @@
+LLM_NAME="ZhipuAI"
+ZHIPUAI_API_KEY="xxxx"
+GLM_MODEL_NAME="glm-3-turbo"
+MIN_RELEVANCE_SCORE=0.3
+BOT_TOPIC="xxxx"
+URL_PREFIX="http://127.0.0.1:7000/"
+USE_PREPROCESS_QUERY=0
+USE_RERANKING=1
+USE_DEBUG=0

env_of_openai

@@ -16,3 +16,14 @@ flask-cors==4.0.0
 diskcache==5.6.3
 html2text==2024.2.26
 langchain-openai==0.1.3
+py3langid==0.2.2
+zhipuai==2.0.1
+pymupdf==1.24.2
+mammoth==1.7.1
+python-pptx==0.6.23
+pandas==2.2.1
+tabulate==0.9.0
+aiofiles==23.2.1
+tokenizers==0.15.2
+onnxruntime==1.16.3
+numpy==1.26.4

env_of_zhipuai

@@ -0,0 +1,100 @@
+import time
+from flask import Blueprint, Flask, request
+from werkzeug.security import generate_password_hash, check_password_hash
+from server.app.utils.decorators import token_required
+from server.app.utils.diskcache_lock import diskcache_lock
+from server.app.utils.sqlite_client import get_db_connection
+from server.app.utils.token_helper import TokenHelper
+from server.logger.logger_config import my_logger as logger
+
+
+account_bp = Blueprint('account_config', __name__, url_prefix='/open_kf_api/account')
+
+
+@account_bp.route('/login', methods=['POST'])
+def login():
+    data = request.json
+    account_name = data.get('account_name')
+    password = data.get('password')
+
+    if not account_name or not password:
+        return {'retcode': -20000, 'message': 'Account name and password are required', 'data': {}}
+
+    conn = None
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+
+        # Check if the account exists and verify the password
+        cur.execute('SELECT id, password_hash FROM t_account_tab WHERE account_name = ?', (account_name,))
+        account = cur.fetchone()
+
+        if account and check_password_hash(account['password_hash'], password):
+            # Generate token with account_name in the payload
+            token = TokenHelper.generate_token(account_name)
+            logger.info(f"Generate token: '{token}'")
+            
+            # Set is_login to 1 and update mtime to the current Unix timestamp
+            try:
+                with diskcache_lock.lock():
+                    cur.execute('UPDATE t_account_tab SET is_login = 1, mtime = ? WHERE account_name = ?', (int(time.time()), account_name,))
+                    conn.commit()
+            except Exception as e:
+                logger.error(f"Process discache_lock exception: {e}")
+                return {'retcode': -30000, 'message': f'An error occurred: {e}', 'data': {}}
+
+            return {'retcode': 0, 'message': 'Login successful', 'data': {'token': token}}
+        else:
+            return {'retcode': -20001, 'message': 'Invalid credentials', 'data': {}}
+    except Exception as e:
+        return {'retcode': -30000, 'message': f'An error occurred during login, exception: {e}', 'data': {}}
+    finally:
+        if conn:
+            conn.close()
+
+
+@account_bp.route('/update_password', methods=['POST'])
+@token_required
+def update_password():
+    data = request.json
+    account_name = data.get('account_name')
+    current_password = data.get('current_password')
+    new_password = data.get('new_password')
+
+    if None in (account_name, current_password, new_password):
+        return {'retcode': -20000, 'message': 'Account name, current password, and new password are required', 'data': {}}
+
+    token_user_id = request.user_payload['user_id']
+    if token_user_id != account_name:
+        logger.error(f"account_name:'{account_name}' does not match with token_user_id: '{token_user_id}'")
+        return {'retcode': -20001, 'message': 'Token is invalid!', 'data': {}}
+
+    conn = None
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+
+        # Check if the account exists and verify the current password
+        cur.execute('SELECT id, password_hash FROM t_account_tab WHERE account_name = ?', (account_name,))
+        account = cur.fetchone()
+
+        if not account or not check_password_hash(account['password_hash'], current_password):
+            logger.error(f"Invalid account_name: '{account_name}' or current_password: '{current_password}'")
+            return {'retcode': -20001, 'message': 'Invalid account name or password', 'data': {}}
+
+        # Update the password
+        new_password_hash = generate_password_hash(new_password, method='pbkdf2:sha256', salt_length=10)
+        try:
+            with diskcache_lock.lock():
+                cur.execute('UPDATE t_account_tab SET password_hash = ?, mtime = ? WHERE account_name = ?', (new_password_hash, int(time.time()), account_name,))
+                conn.commit()
+        except Exception as e:
+            logger.error(f"Process discache_lock exception: {e}")
+            return {'retcode': -30000, 'message': f'An error occurred: {e}', 'data': {}}
+
+        return {'retcode': 0, 'message': 'Password updated successfully', 'data': {}}
+    except Exception as e:
+        return {'retcode': -20001, 'message': f'An error occurred: {e}', 'data': {}}
+    finally:
+        if conn:
+            conn.close()

rag_gpt_app.py

@@ -0,0 +1,23 @@
+from flask import Blueprint, request
+from server.logger.logger_config import my_logger as logger
+from server.app.utils.token_helper import TokenHelper
+
+
+auth_bp = Blueprint('auth', __name__, url_prefix='/open_kf_api/auth')
+
+
+@auth_bp.route('/get_token', methods=['POST'])
+def get_token():
+    data = request.json
+    user_id = data.get('user_id')
+    if not user_id:
+        return {'retcode': -20000, 'message': 'user_id is required', 'data': {}}
+
+    try:
+        # generate token
+        token = TokenHelper.generate_token(user_id)
+        logger.success(f"Generate token: '{token}' with user_id: '{user_id}'")
+        return {"retcode": 0, "message": "success", "data": {"token": token}}
+    except Exception as e:
+        logger.error(f"Generate token with user_id: '{user_id}' is failed, the exception is {e}")
+        return {'retcode': -20001, 'message': str(e), 'data': {}}

requirements.txt

@@ -0,0 +1,136 @@
+import json
+import time
+from flask import Blueprint, Flask, request
+from server.app.utils.decorators import token_required
+from server.app.utils.sqlite_client import get_db_connection
+from server.app.utils.diskcache_client import diskcache_client
+from server.app.utils.diskcache_lock import diskcache_lock
+from server.logger.logger_config import my_logger as logger
+
+
+bot_config_bp = Blueprint('bot_config', __name__, url_prefix='/open_kf_api/bot_config')
+
+
+@bot_config_bp.route('/get_bot_setting', methods=['POST'])
+def get_bot_setting():
+    """Retrieve bot setting, first trying Cache and falling back to DB if not found."""
+    try:
+        # Attempt to retrieve the setting from Cache
+        key = "open_kf:bot_setting"
+        setting_cache = diskcache_client.get(key)
+        if setting_cache:
+            setting_data = json.loads(setting_cache)
+            return {'retcode': 0, 'message': 'Success', 'data': {'config': setting_data}}
+        else:
+            logger.warning(f"could not find '{key}' in Cache!")
+    except Exception as e:
+        logger.error(f"Error retrieving setting from Cache, excpetion:{e}")
+        # Just ignore Cache error
+        #return {'retcode': -30000, 'message': f'An error occurred: {str(e)}', 'data': {}}
+
+    conn = None
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+
+        cur.execute('SELECT * FROM t_bot_setting_tab LIMIT 1')
+        setting = cur.fetchone()
+        if setting:
+            setting = dict(setting)
+            # Process and return the setting details
+            setting_data = {k: json.loads(v) if k in ['initial_messages', 'suggested_messages'] else v for k, v in setting.items()}
+
+            # Add bot setting into Cache
+            try:
+                key = "open_kf:bot_setting"
+                diskcache_client.set(key, json.dumps(setting_data))
+            except Exception as e:
+                logger.error(f"Add bot setting into Cache is failed, the exception is {e}")
+                # Just ignore Reids error
+
+            return {'retcode': 0, 'message': 'Success', 'data': {'config': setting_data}}
+        else:
+            logger.warning(f"No setting found")
+            return {'retcode': -20001, 'message': 'No setting found', 'data': {}}
+    except Exception as e:
+        logger.error(f"Error retrieving setting: {e}")
+        return {'retcode': -30000, 'message': f'An error occurred: {e}', 'data': {}}
+    finally:
+        if conn:
+            conn.close()
+
+
+@bot_config_bp.route('/update_bot_setting', methods=['POST'])
+@token_required
+def update_bot_setting():
+    data = request.json
+    # Extract and validate all required fields
+    setting_id = data.get('id')
+    initial_messages = data.get('initial_messages')
+    suggested_messages = data.get('suggested_messages')
+    bot_name = data.get('bot_name')
+    bot_avatar = data.get('bot_avatar')
+    chat_icon = data.get('chat_icon')
+    placeholder = data.get('placeholder')
+    model = data.get('model')
+
+    # Check for the presence of all required fields
+    if None in (setting_id, initial_messages, suggested_messages, bot_name, bot_avatar, chat_icon, placeholder, model):
+        return {'retcode': -20000, 'message': 'All fields are required', 'data': {}}
+
+    conn = None
+    try:
+        conn = get_db_connection()
+        cur = conn.cursor()
+
+        # Check if the setting with provided ID exists
+        cur.execute('SELECT id FROM t_bot_setting_tab WHERE id = ?', (setting_id,))
+        if not cur.fetchone():
+            logger.error(f"No setting found")
+            return {'retcode': -20001, 'message': 'Setting not found', 'data': {}}
+
+        # Convert lists to JSON strings for storage
+        initial_messages_json = json.dumps(initial_messages)
+        suggested_messages_json = json.dumps(suggested_messages)
+
+        # Update bot setting in DB
+        timestamp = int(time.time())
+        try:
+            with diskcache_lock.lock():
+                cur.execute('''
+                    UPDATE t_bot_setting_tab
+                    SET initial_messages = ?, suggested_messages = ?, bot_name = ?, bot_avatar = ?, chat_icon = ?, placeholder = ?, model = ?, mtime = ?
+                    WHERE id = ?
+                ''', (initial_messages_json, suggested_messages_json, bot_name, bot_avatar, chat_icon, placeholder, model, timestamp, setting_id))
+                conn.commit()
+        except Exception as e:
+            logger.error(f"Process discache_lock exception: {e}")
+            return {'retcode': -30000, 'message': f'An error occurred: {e}', 'data': {}}
+
+        # Update bot setting in Cache
+        try:
+            key = "open_kf:bot_setting"
+            bot_setting = {
+                'id': setting_id,
+                'initial_messages': initial_messages,
+                'suggested_messages': suggested_messages,
+                'bot_name': bot_name,
+                'bot_avatar': bot_avatar,
+                'chat_icon': chat_icon,
+                'placeholder': placeholder,
+                'model': model,
+                'ctime': timestamp,
+                'mtime': timestamp
+            }
+            diskcache_client.set(key, json.dumps(bot_setting))
+        except Exception as e:
+            logger.error(f"Ppdate bot seeting in Cache is failed, the exception is {e}")
+            return {'retcode': -20001, 'message': f'An error occurred: {e}', 'data': {}}
+
+        return {'retcode': 0, 'message': 'Settings updated successfully', 'data': {}}
+    except Exception as e:
+        logger.error(f"Error updating setting in DB: {e}")
+        return {'retcode': -30000, 'message': f'An error occurred: {e}', 'data': {}}
+    finally:
+        if conn:
+            conn.close()

server/__init__.py

@@ -0,0 +1,38 @@
+from datetime import datetime
+import os
+import uuid
+from flask import Blueprint, Flask, request
+from werkzeug.utils import secure_filename
+from server.constant.constants import STATIC_DIR, MEDIA_DIR
+from server.app.utils.decorators import token_required
+from server.logger.logger_config import my_logger as logger
+
+
+URL_PREFIX = os.getenv('URL_PREFIX')
+
+common_bp = Blueprint('common', __name__, url_prefix='/open_kf_api/common')
+
+
+@common_bp.route('/upload_picture', methods=['POST'])
+@token_required
+def upload_picture():
+    picture_file = request.files.get('picture_file')
+    if not picture_file:
+        logger.error("Missing required parameters picture_file")
+        return {'retcode': -20000, 'message': 'Missing required parameters picture_file', data:{}}
+
+    try:
+        original_filename = secure_filename(picture_file.filename)
+
+        day_folder = datetime.now().strftime("%Y_%m_%d")
+        unique_folder = str(uuid.uuid4())
+        save_directory = os.path.join(STATIC_DIR, MEDIA_DIR, day_folder, unique_folder)
+        os.makedirs(save_directory, exist_ok=True)
+
+        image_path = os.path.join(save_directory, original_filename)
+        picture_file.save(image_path)
+        picture_url  = f"{URL_PREFIX}{MEDIA_DIR}/{day_folder}/{unique_folder}/{original_filename}"
+        return {'retcode': 0, 'message': 'upload picture success', 'data': {'picture_url': picture_url}}
+    except Exception as e:
+        logger.error(f"An error occureed: {str(e)}")
+        return {'retcode': -30000, 'message': f'An error occurred: {str(e)}', 'data': {}}