Add feedback buttons with optional setting (#396)

Co-authored-by: Sarah Widder <[email protected]>
Co-authored-by: Ian Seabock (Centific Technologies Inc) <[email protected]>

.env.sample

@@ -36,4 +36,6 @@ AZURE_COSMOSDB_MONGO_VCORE_CONNECTION_STRING=
 AZURE_COSMOSDB_MONGO_VCORE_CONTAINER=
 AZURE_COSMOSDB_MONGO_VCORE_INDEX=
 AZURE_COSMOSDB_MONGO_VCORE_CONTENT_COLUMNS=
-AZURE_COSMOSDB_MONGO_VCORE_VECTOR_COLUMNS=
+AZURE_COSMOSDB_MONGO_VCORE_VECTOR_COLUMNS=
+AZURE_COSMOSDB_ENABLE_FEEDBACK=False
+AUTH_ENABLED=False

README.md

@@ -79,6 +79,12 @@ To enable chat history, you will need to set up CosmosDB resources. The ARM temp
 
 As above, start the app with `start.cmd`, then visit the local running app at http://127.0.0.1:5000.
 
+#### Local Setup: Enable Message Feedback
+To enable message feedback, you will need to set up CosmosDB resources. Then specify these additional environment variable:
+
+/.env
+- `AZURE_COSMOSDB_ENABLE_FEEDBACK=True`
+
 #### Deploy with the Azure CLI
 **NOTE**: If you've made code changes, be sure to **build the app code** with `start.cmd` or `start.sh` before you deploy, otherwise your changes will not be picked up. If you've updated any files in the `frontend` folder, make sure you see updates to the files in the `static` folder before you deploy.
 

app.py

@@ -4,6 +4,7 @@
 import requests
 import openai
 import copy
+import uuid
 from azure.identity import DefaultAzureCredential
 from base64 import b64encode
 from flask import Flask, Response, request, jsonify, send_from_directory
@@ -97,6 +98,7 @@ def assets(path):
 AZURE_COSMOSDB_ACCOUNT = os.environ.get("AZURE_COSMOSDB_ACCOUNT")
 AZURE_COSMOSDB_CONVERSATIONS_CONTAINER = os.environ.get("AZURE_COSMOSDB_CONVERSATIONS_CONTAINER")
 AZURE_COSMOSDB_ACCOUNT_KEY = os.environ.get("AZURE_COSMOSDB_ACCOUNT_KEY")
+AZURE_COSMOSDB_ENABLE_FEEDBACK = os.environ.get("AZURE_COSMOSDB_ENABLE_FEEDBACK", "false").lower() == "true"
 
 # Elasticsearch Integration Settings
 ELASTICSEARCH_ENDPOINT = os.environ.get("ELASTICSEARCH_ENDPOINT")
@@ -114,9 +116,13 @@ def assets(path):
 ELASTICSEARCH_EMBEDDING_MODEL_ID = os.environ.get("ELASTICSEARCH_EMBEDDING_MODEL_ID")
 
 # Frontend Settings via Environment Variables
-AUTH_ENABLED = os.environ.get("AUTH_ENABLED", "true").lower()
-frontend_settings = { "auth_enabled": AUTH_ENABLED }
+AUTH_ENABLED = os.environ.get("AUTH_ENABLED", "true").lower() == "true"
+frontend_settings = { 
+    "auth_enabled": AUTH_ENABLED, 
+    "feedback_enabled": AZURE_COSMOSDB_ENABLE_FEEDBACK and AZURE_COSMOSDB_DATABASE not in [None, ""],
+}
 
+message_uuid = ""
 
 # Initialize a CosmosDB client with AAD auth and containers for Chat History
 cosmos_conversation_client = None
@@ -133,7 +139,8 @@ def assets(path):
             cosmosdb_endpoint=cosmos_endpoint, 
             credential=credential, 
             database_name=AZURE_COSMOSDB_DATABASE,
-            container_name=AZURE_COSMOSDB_CONVERSATIONS_CONTAINER
+            container_name=AZURE_COSMOSDB_CONVERSATIONS_CONTAINER,
+            enable_message_feedback = AZURE_COSMOSDB_ENABLE_FEEDBACK
         )
     except Exception as e:
         logging.exception("Exception in CosmosDB initialization", e)
@@ -257,7 +264,7 @@ def prepare_body_headers_with_data(request):
                         "vectorFields": parse_multi_columns(AZURE_SEARCH_VECTOR_COLUMNS) if AZURE_SEARCH_VECTOR_COLUMNS else []
                     },
                     "inScope": True if AZURE_SEARCH_ENABLE_IN_DOMAIN.lower() == "true" else False,
-                    "topNDocuments": AZURE_SEARCH_TOP_K,
+                    "topNDocuments": int(AZURE_SEARCH_TOP_K),
                     "queryType": query_type,
                     "semanticConfiguration": AZURE_SEARCH_SEMANTIC_SEARCH_CONFIG if AZURE_SEARCH_SEMANTIC_SEARCH_CONFIG else "",
                     "roleInformation": AZURE_OPENAI_SYSTEM_MESSAGE,
@@ -285,7 +292,7 @@ def prepare_body_headers_with_data(request):
                         "vectorFields": parse_multi_columns(AZURE_COSMOSDB_MONGO_VCORE_VECTOR_COLUMNS) if AZURE_COSMOSDB_MONGO_VCORE_VECTOR_COLUMNS else []
                     },
                     "inScope": True if AZURE_COSMOSDB_MONGO_VCORE_ENABLE_IN_DOMAIN.lower() == "true" else False,
-                    "topNDocuments": AZURE_COSMOSDB_MONGO_VCORE_TOP_K,
+                    "topNDocuments": int(AZURE_COSMOSDB_MONGO_VCORE_TOP_K),
                     "strictness": int(AZURE_COSMOSDB_MONGO_VCORE_STRICTNESS),
                     "queryType": query_type,
                     "roleInformation": AZURE_OPENAI_SYSTEM_MESSAGE
@@ -387,7 +394,7 @@ def stream_with_data(body, headers, endpoint, history_metadata={}):
 
                     if 'error' in lineJson:
                         yield format_as_ndjson(lineJson)
-                    response["id"] = lineJson["id"]
+                    response["id"] = message_uuid
                     response["model"] = lineJson["model"]
                     response["created"] = lineJson["created"]
                     response["object"] = lineJson["object"]
@@ -520,7 +527,7 @@ def stream_without_data(response, history_metadata={}):
             responseText = deltaText
 
         response_obj = {
-            "id": line["id"],
+            "id": message_uuid,
             "model": line["model"],
             "created": line["created"],
             "object": line["object"],
@@ -570,7 +577,7 @@ def conversation_without_data(request_body):
 
     if not SHOULD_STREAM:
         response_obj = {
-            "id": response,
+            "id": message_uuid,
             "model": response.model,
             "created": response.created,
             "object": response.object,
@@ -607,6 +614,8 @@ def conversation_internal(request_body):
 ## Conversation History API ## 
 @app.route("/history/generate", methods=["POST"])
 def add_conversation():
+    global message_uuid
+    message_uuid = str(uuid.uuid4())
     authenticated_user = get_authenticated_user_details(request_headers=request.headers)
     user_id = authenticated_user['user_principal_id']
 
@@ -632,6 +641,7 @@ def add_conversation():
         messages = request.json["messages"]
         if len(messages) > 0 and messages[-1]['role'] == "user":
             cosmos_conversation_client.create_message(
+                uuid=str(uuid.uuid4()),
                 conversation_id=conversation_id,
                 user_id=user_id,
                 input_message=messages[-1]
@@ -674,12 +684,14 @@ def update_conversation():
             if len(messages) > 1 and messages[-2].get('role', None) == "tool":
                 # write the tool message first
                 cosmos_conversation_client.create_message(
+                    uuid=str(uuid.uuid4()),
                     conversation_id=conversation_id,
                     user_id=user_id,
                     input_message=messages[-2]
                 )
             # write the assistant message
             cosmos_conversation_client.create_message(
+                uuid=message_uuid,
                 conversation_id=conversation_id,
                 user_id=user_id,
                 input_message=messages[-1]
@@ -695,6 +707,33 @@ def update_conversation():
         logging.exception("Exception in /history/update")
         return jsonify({"error": str(e)}), 500
 
+@app.route("/history/message_feedback", methods=["POST"])
+def update_message():
+    authenticated_user = get_authenticated_user_details(request_headers=request.headers)
+    user_id = authenticated_user['user_principal_id']
+
+    ## check request for message_id
+    message_id = request.json.get("message_id", None)
+    message_feedback = request.json.get("message_feedback", None)
+    try:
+        if not message_id:
+            return jsonify({"error": "message_id is required"}), 400
+
+        if not message_feedback:
+            return jsonify({"error": "message_feedback is required"}), 400
+
+        ## update the message in cosmos
+        updated_message = cosmos_conversation_client.update_message_feedback(user_id, message_id, message_feedback)
+        if updated_message:
+            return jsonify({"message": f"Successfully updated message with feedback {message_feedback}", "message_id": message_id}), 200
+        else:
+            return jsonify({"error": f"Unable to update message {message_id}. It either does not exist or the user does not have access to it."}), 404
+
+    except Exception as e:
+        logging.exception("Exception in /history/message_feedback")
+        return jsonify({"error": str(e)}), 500
+
+
 @app.route("/history/delete", methods=["DELETE"])
 def delete_conversation():
     ## get the user id from the request headers
@@ -754,7 +793,7 @@ def get_conversation():
     conversation_messages = cosmos_conversation_client.get_messages(user_id, conversation_id)
 
     ## format the messages in the bot frontend format
-    messages = [{'id': msg['id'], 'role': msg['role'], 'content': msg['content'], 'createdAt': msg['createdAt']} for msg in conversation_messages]
+    messages = [{'id': msg['id'], 'role': msg['role'], 'content': msg['content'], 'createdAt': msg['createdAt'], 'feedback': msg.get('feedback')} for msg in conversation_messages]
 
     return jsonify({"conversation_id": conversation_id, "messages": messages}), 200
 

backend/history/cosmosdbservice.py

@@ -7,14 +7,15 @@
 
 class CosmosConversationClient():
 
-    def __init__(self, cosmosdb_endpoint: str, credential: any, database_name: str, container_name: str):
+    def __init__(self, cosmosdb_endpoint: str, credential: any, database_name: str, container_name: str, enable_message_feedback: bool = False):
         self.cosmosdb_endpoint = cosmosdb_endpoint
         self.credential = credential
         self.database_name = database_name
         self.container_name = container_name
         self.cosmosdb_client = CosmosClient(self.cosmosdb_endpoint, credential=credential)
         self.database_client = self.cosmosdb_client.get_database_client(database_name)
         self.container_client = self.database_client.get_container_client(container_name)
+        self.enable_message_feedback = enable_message_feedback
 
     def ensure(self):
         try:
@@ -111,9 +112,9 @@ def get_conversation(self, user_id, conversation_id):
         else:
             return conversation[0]
 
-    def create_message(self, conversation_id, user_id, input_message: dict):
+    def create_message(self, uuid, conversation_id, user_id, input_message: dict):
         message = {
-            'id': str(uuid.uuid4()),
+            'id': uuid,
             'type': 'message',
             'userId' : user_id,
             'createdAt': datetime.utcnow().isoformat(),
@@ -122,6 +123,9 @@ def create_message(self, conversation_id, user_id, input_message: dict):
             'role': input_message['role'],
             'content': input_message['content']
         }
+
+        if self.enable_message_feedback:
+            message['feedback'] = ''
 
         resp = self.container_client.upsert_item(message)  
         if resp:
@@ -133,7 +137,14 @@ def create_message(self, conversation_id, user_id, input_message: dict):
         else:
             return False
 
-
+    def update_message_feedback(self, user_id, message_id, feedback):
+        message = self.container_client.read_item(item=message_id, partition_key=user_id)
+        if message:
+            message['feedback'] = feedback
+            resp = self.container_client.upsert_item(message)
+            return resp
+        else:
+            return False
 
     def get_messages(self, user_id, conversation_id):
         parameters = [

frontend/src/api/api.ts

@@ -94,6 +94,7 @@ export const historyRead = async (convId: string): Promise<ChatMessage[]> => {
                     role: msg.role,
                     date: msg.createdAt,
                     content: msg.content,
+                    feedback: msg.feedback ?? undefined
                 }
                 messages.push(message)
             });
@@ -309,3 +310,28 @@ export const frontendSettings = async (): Promise<Response | null> => {
 
     return response
 }
+export const historyMessageFeedback = async (messageId: string, feedback: string): Promise<Response> => {
+    const response = await fetch("/history/message_feedback", {
+        method: "POST",
+        body: JSON.stringify({
+            message_id: messageId,
+            message_feedback: feedback
+        }),
+        headers: {
+            "Content-Type": "application/json"
+        },
+    })
+    .then((res) => {
+        return res
+    })
+    .catch((err) => {
+        console.error("There was an issue logging feedback.");
+        let errRes: Response = {
+            ...new Response,
+            ok: false,
+            status: 500,
+        }
+        return errRes;
+    })
+    return response;
+}

frontend/src/api/models.ts

@@ -2,6 +2,8 @@ export type AskResponse = {
     answer: string;
     citations: Citation[];
     error?: string;
+    message_id?: string;
+    feedback?: Feedback;
 };
 
 export type Citation = {
@@ -26,6 +28,7 @@ export type ChatMessage = {
     content: string;
     end_turn?: boolean;
     date: string;
+    feedback?: Feedback;
 };
 
 export type Conversation = {
@@ -96,4 +99,21 @@ export type ErrorMessage = {
 
 export type FrontendSettings = {
     auth_enabled?: string | null;
+    feedback_enabled?: string | null;
+}
+
+export enum Feedback {
+    Neutral = "neutral",
+    Positive = "positive",
+    Negative = "negative",
+    MissingCitation = "missing_citation",
+    WrongCitation = "wrong_citation",
+    OutOfScope = "out_of_scope",
+    InaccurateOrIrrelevant = "inaccurate_or_irrelevant",
+    OtherUnhelpful = "other_unhelpful",
+    HateSpeech = "hate_speech",
+    Violent = "violent",
+    Sexual = "sexual",
+    Manipulative = "manipulative",
+    OtherHarmful = "other_harmlful"
 }

frontend/src/components/Answer/Answer.module.css

@@ -26,6 +26,10 @@
     overflow-x: auto;
 }
 
+.answerHeader {
+    position: relative;
+}
+
 .answerFooter {
     display: flex;
     flex-flow: row nowrap;