Skip to content

Commit

Permalink
Feat 25501845 improve chat app response latency (#355)
Browse files Browse the repository at this point in the history 
Co-authored-by: Ian Seabock (Centific Technologies Inc) <[email protected]>
  • Loading branch information
@iseabock
iseabock and Ian Seabock (Centific Technologies Inc) authored Nov 2, 2023
1 parent 6e644c1 commit edcc5b8
Show file tree
Hide file tree
Showing 7 changed files with 178 additions and 154 deletions.
37 changes: 20 additions & 17 deletions app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,21 +213,20 @@ def prepare_body_headers_with_data(request):

def stream_with_data(body, headers, endpoint, history_metadata={}):
s = requests.Session()
response = {
"id": "",
"model": "",
"created": 0,
"object": "",
"choices": [{
"messages": []
}],
"apim-request-id": "",
'history_metadata': history_metadata
}
try:
with s.post(endpoint, json=body, headers=headers, stream=True) as r:
apimRequestId = r.headers.get('apim-request-id')
for line in r.iter_lines(chunk_size=10):
response = {
"id": "",
"model": "",
"created": 0,
"object": "",
"choices": [{
"messages": []
}],
"apim-request-id": "",
'history_metadata': history_metadata
}
if line:
if AZURE_OPENAI_PREVIEW_API_VERSION == '2023-06-01-preview':
lineJson = json.loads(line.lstrip(b'data:').decode('utf-8'))
Expand All @@ -244,23 +243,27 @@ def stream_with_data(body, headers, endpoint, history_metadata={}):
response["model"] = lineJson["model"]
response["created"] = lineJson["created"]
response["object"] = lineJson["object"]
response["apim-request-id"] = apimRequestId
response["apim-request-id"] = r.headers.get('apim-request-id')

role = lineJson["choices"][0]["messages"][0]["delta"].get("role")

if role == "tool":
response["choices"][0]["messages"].append(lineJson["choices"][0]["messages"][0]["delta"])
yield format_as_ndjson(response)
elif role == "assistant":
response["choices"][0]["messages"].append({
"role": "assistant",
"content": ""
})
yield format_as_ndjson(response)
else:
deltaText = lineJson["choices"][0]["messages"][0]["delta"]["content"]
if deltaText != "[DONE]":
response["choices"][0]["messages"][1]["content"] += deltaText

yield format_as_ndjson(response)
response["choices"][0]["messages"].append({
"role": "assistant",
"content": deltaText
})
yield format_as_ndjson(response)
except Exception as e:
yield format_as_ndjson({"error" + str(e)})

Expand Down Expand Up @@ -364,7 +367,7 @@ def stream_without_data(response, history_metadata={}):
else:
deltaText = ""
if deltaText and deltaText != "[DONE]":
responseText += deltaText
responseText = deltaText

response_obj = {
"id": line["id"],
Expand Down
71 changes: 46 additions & 25 deletions frontend/src/pages/chat/Chat.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import ReactMarkdown from "react-markdown";
import remarkGfm from 'remark-gfm'
import rehypeRaw from "rehype-raw";
import uuid from 'react-uuid';
import { isEmpty } from "lodash-es";

import styles from "./Chat.module.css";
import Azure from "../../assets/Azure.svg";
Expand Down Expand Up @@ -67,6 +68,8 @@ const Chat = () => {
styles: { main: { maxWidth: 450 } },
}

const [ASSISTANT, TOOL, ERROR] = ["assistant", "tool", "error"]

useEffect(() => {
if(appStateContext?.state.isCosmosDBAvailable?.status === CosmosDBStatus.NotWorking && appStateContext.state.chatHistoryLoadingState === ChatHistoryLoadingState.Fail && hideErrorDialog){
let subtitle = `${appStateContext.state.isCosmosDBAvailable.status}. Please contact the site administrator.`
Expand Down Expand Up @@ -95,6 +98,30 @@ const Chat = () => {
}
}

let assistantMessage = {} as ChatMessage
let toolMessage = {} as ChatMessage
let assistantContent = ""

const processResultMessage = (resultMessage: ChatMessage, userMessage: ChatMessage, conversationId?: string) => {
if (resultMessage.role === ASSISTANT) {
assistantContent += resultMessage.content
assistantMessage = resultMessage
assistantMessage.content = assistantContent
}

if (resultMessage.role === TOOL) toolMessage = resultMessage

if (!conversationId) {
isEmpty(toolMessage) ?
setMessages([...messages, userMessage, assistantMessage]) :
setMessages([...messages, userMessage, toolMessage, assistantMessage]);
} else {
isEmpty(toolMessage) ?
setMessages([...messages, assistantMessage]) :
setMessages([...messages, toolMessage, assistantMessage]);
}
}

const makeApiRequestWithoutCosmosDB = async (question: string, conversationId?: string) => {
setIsLoading(true);
setShowLoadingMessage(true);
Expand Down Expand Up @@ -133,8 +160,7 @@ const Chat = () => {
setMessages(conversation.messages)

const request: ConversationRequest = {
messages: [...conversation.messages.filter((answer) => answer.role !== "error")]
// messages: [...conversation.messages.filter((answer) => answer.role === "error")]
messages: [...conversation.messages.filter((answer) => answer.role !== ERROR)]
};

let result = {} as ChatResponse;
Expand All @@ -160,19 +186,17 @@ const Chat = () => {
obj.date = new Date().toISOString();
})
setShowLoadingMessage(false);
if(!conversationId){
setMessages([...messages, userMessage, ...result.choices[0].messages]);
}else{
setMessages([...messages, ...result.choices[0].messages]);
}
result.choices[0].messages.forEach((resultObj) => {
processResultMessage(resultObj, userMessage, conversationId);
})
runningText = "";
}
catch { }
});
}
conversation.messages.push(...result.choices[0].messages)
conversation.messages.push(toolMessage, assistantMessage)
appStateContext?.dispatch({ type: 'UPDATE_CURRENT_CHAT', payload: conversation });
setMessages([...messages, ...result.choices[0].messages]);
setMessages([...messages, toolMessage, assistantMessage]);
}

} catch ( e ) {
Expand All @@ -186,7 +210,7 @@ const Chat = () => {
}
let errorChatMsg: ChatMessage = {
id: uuid(),
role: "error",
role: ERROR,
content: errorMessage,
date: new Date().toISOString()
}
Expand Down Expand Up @@ -233,12 +257,12 @@ const Chat = () => {
}else{
conversation.messages.push(userMessage);
request = {
messages: [...conversation.messages.filter((answer) => answer.role !== "error")]
messages: [...conversation.messages.filter((answer) => answer.role !== ERROR)]
};
}
}else{
request = {
messages: [userMessage].filter((answer) => answer.role !== "error")
messages: [userMessage].filter((answer) => answer.role !== ERROR)
};
setMessages(request.messages)
}
Expand All @@ -248,7 +272,7 @@ const Chat = () => {
if(!response?.ok){
let errorChatMsg: ChatMessage = {
id: uuid(),
role: "error",
role: ERROR,
content: "There was an error generating a response. Chat history can't be saved at this time. If the problem persists, please contact the site administrator.",
date: new Date().toISOString()
}
Expand Down Expand Up @@ -294,11 +318,9 @@ const Chat = () => {
obj.date = new Date().toISOString();
})
setShowLoadingMessage(false);
if(!conversationId){
setMessages([...messages, userMessage, ...result.choices[0].messages]);
}else{
setMessages([...messages, ...result.choices[0].messages]);
}
result.choices[0].messages.forEach((resultObj) => {
processResultMessage(resultObj, userMessage, conversationId);
})
runningText = "";
}
catch { }
Expand All @@ -315,15 +337,15 @@ const Chat = () => {
abortFuncs.current = abortFuncs.current.filter(a => a !== abortController);
return;
}
resultConversation.messages.push(...result.choices[0].messages);
resultConversation.messages.push(toolMessage, assistantMessage)
}else{
resultConversation = {
id: result.history_metadata.conversation_id,
title: result.history_metadata.title,
messages: [userMessage],
date: result.history_metadata.date
}
resultConversation.messages.push(...result.choices[0].messages);
resultConversation.messages.push(toolMessage, assistantMessage)
}
if(!resultConversation){
setIsLoading(false);
Expand All @@ -332,7 +354,7 @@ const Chat = () => {
return;
}
appStateContext?.dispatch({ type: 'UPDATE_CURRENT_CHAT', payload: resultConversation });
setMessages([...messages, ...result.choices[0].messages]);
setMessages([...messages, toolMessage, assistantMessage]);
}

} catch ( e ) {
Expand All @@ -346,7 +368,7 @@ const Chat = () => {
}
let errorChatMsg: ChatMessage = {
id: uuid(),
role: "error",
role: ERROR,
content: errorMessage,
date: new Date().toISOString()
}
Expand Down Expand Up @@ -436,7 +458,6 @@ const Chat = () => {

useEffect(() => {
if (appStateContext?.state.currentChat) {

setMessages(appStateContext.state.currentChat.messages)
}else{
setMessages([])
Expand All @@ -461,7 +482,7 @@ const Chat = () => {
let errorMessage = "An error occurred. Answers can't be saved at this time. If the problem persists, please contact the site administrator.";
let errorChatMsg: ChatMessage = {
id: uuid(),
role: "error",
role: ERROR,
content: errorMessage,
date: new Date().toISOString()
}
Expand Down Expand Up @@ -574,7 +595,7 @@ const Chat = () => {
}}
onCitationClicked={c => onShowCitation(c)}
/>
</div> : answer.role === "error" ? <div className={styles.chatMessageError}>
</div> : answer.role === ERROR ? <div className={styles.chatMessageError}>
<Stack horizontal className={styles.chatMessageErrorContent}>
<ErrorCircleRegular className={styles.errorIcon} style={{color: "rgba(182, 52, 67, 1)"}} />
<span>Error</span>
Expand Down
Loading

0 comments on commit edcc5b8

Please sign in to comment.