Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplify chat message streaming in chat template #6120

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Simplify chat message streaming in chat template #6120

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0ca5bda
Render chat response updates directly
MackinnonBuck Mar 15, 2025
f71526d
Merge branch 'main' into mbuck/simplify-chat-streaming
MackinnonBuck Mar 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 21 additions & 18 deletions ...tes/src/ChatWithCustomData/ChatWithCustomData.Web-CSharp/Components/Pages/Chat/Chat.razor
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

<ChatHeader OnNewChat="@ResetConversationAsync" />

<ChatMessageList Messages="@messages" InProgressMessage="@currentResponseMessage">
<ChatMessageList Messages="@messages" @ref="chatMessageList">
<NoMessagesContent>
<div>To get started, try asking about these example documents. You can replace these with your own data and replace this message.</div>
<ChatCitation File="Example_Emergency_Survival_Kit.pdf"/>
Expand All @@ -18,8 +18,8 @@
</ChatMessageList>

<div class="chat-container">
<ChatSuggestions OnSelected="@AddUserMessageAsync" @ref="@chatSuggestions" />
<ChatInput OnSend="@AddUserMessageAsync" @ref="@chatInput" />
<ChatSuggestions OnSelected="@AddUserMessageAsync" @ref="chatSuggestions" />
<ChatInput OnSend="@AddUserMessageAsync" @ref="chatInput" />
<SurveyPrompt /> @* Remove this line to eliminate the template survey message *@
</div>

Expand All @@ -42,8 +42,9 @@

private readonly ChatOptions chatOptions = new();
private readonly List<ChatMessage> messages = new();
private readonly List<ChatResponseUpdate> currentResponseUpdates = new();
private CancellationTokenSource? currentResponseCancellation;
private ChatMessage? currentResponseMessage;
private ChatMessageList? chatMessageList;
private ChatInput? chatInput;
private ChatSuggestions? chatSuggestions;

Expand Down Expand Up @@ -72,33 +73,35 @@
messages.AddMessages(response);
#else*@
// Stream and display a new response from the IChatClient
var responseText = new TextContent("");
currentResponseMessage = new ChatMessage(ChatRole.Assistant, [responseText]);
currentResponseCancellation = new();
await foreach (var update in ChatClient.GetStreamingResponseAsync([.. messages], chatOptions, currentResponseCancellation.Token))
await foreach (var update in ChatClient.GetStreamingResponseAsync(messages, chatOptions, currentResponseCancellation.Token))
{
messages.AddMessages(update, filter: c => c is not TextContent);
responseText.Text += update.Text;
ChatMessageItem.NotifyChanged(currentResponseMessage);
currentResponseUpdates.Add(update);
chatMessageList?.AddStreamingResponseUpdate(update);
}

// Store the final response in the conversation, and begin getting suggestions
messages.Add(currentResponseMessage!);
currentResponseMessage = null;
CompleteCurrentResponse();
@*#endif*@
chatSuggestions?.Update(messages);
}

@*#if (!IsOllama)*@
private void CompleteCurrentResponse()
{
messages.AddMessages(currentResponseUpdates);
currentResponseUpdates.Clear();
chatMessageList?.ClearStreamingResponseUpdates();
}
@*#endif*@

private void CancelAnyCurrentResponse()
{
@*#if (!IsOllama)*@
// If a response was cancelled while streaming, include it in the conversation so it's not lost
if (currentResponseMessage is not null)
{
messages.Add(currentResponseMessage);
}

CompleteCurrentResponse();
@*#endif*@
currentResponseCancellation?.Cancel();
currentResponseMessage = null;
}

private async Task ResetConversationAsync()
Expand Down
54 changes: 54 additions & 0 deletions ...omData/ChatWithCustomData.Web-CSharp/Components/Pages/Chat/ChatAssistantContentItem.razor
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
@if (text is { Length: > 0 })
{
<div class="assistant-message">
<div>
<div class="assistant-message-icon">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor">
<path stroke-linecap="round" stroke-linejoin="round" d="M12 18v-5.25m0 0a6.01 6.01 0 0 0 1.5-.189m-1.5.189a6.01 6.01 0 0 1-1.5-.189m3.75 7.478a12.06 12.06 0 0 1-4.5 0m3.75 2.383a14.406 14.406 0 0 1-3 0M14.25 18v-.192c0-.983.658-1.823 1.508-2.316a7.5 7.5 0 1 0-7.517 0c.85.493 1.509 1.333 1.509 2.316V18" />
</svg>
</div>
</div>
<div class="assistant-message-header">Assistant</div>
<div class="assistant-message-text">
<assistant-message markdown="@text"></assistant-message>

@TextContentFooter
</div>
</div>
}
else if (Content is FunctionCallContent { Name: "Search" } fcc && fcc.Arguments?.TryGetValue("searchPhrase", out var searchPhrase) is true)
{
<div class="assistant-search">
<div class="assistant-search-icon">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor">
<path stroke-linecap="round" stroke-linejoin="round" d="m21 21-5.197-5.197m0 0A7.5 7.5 0 1 0 5.196 5.196a7.5 7.5 0 0 0 10.607 10.607Z" />
</svg>
</div>
<div class="assistant-search-content">
Searching:
<span class="assistant-search-phrase">@searchPhrase</span>
@if (fcc.Arguments?.TryGetValue("filenameFilter", out var filenameObj) is true && filenameObj is string filename && !string.IsNullOrEmpty(filename))
{
<text> in </text><span class="assistant-search-phrase">@filename</span>
}
</div>
</div>
}

@code {
private string? text;

[Parameter]
public AIContent? Content { get; set; }

[Parameter]
public string? Text { get; set; }

[Parameter]
public RenderFragment? TextContentFooter { get; set; }

protected override void OnParametersSet()
{
text = Text ?? (Content as TextContent)?.Text;
}
}
101 changes: 101 additions & 0 deletions ...ta/ChatWithCustomData.Web-CSharp/Components/Pages/Chat/ChatAssistantContentItem.razor.css
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
.assistant-message, .assistant-search {
display: grid;
grid-template-rows: min-content;
grid-template-columns: 2rem minmax(0, 1fr);
gap: 0.25rem;
}

.assistant-message-header {
font-weight: 600;
}

.assistant-message-text {
grid-column-start: 2;
}

.assistant-message-icon {
display: flex;
justify-content: center;
align-items: center;
border-radius: 9999px;
width: 1.5rem;
height: 1.5rem;
color: #ffffff;
background: #9b72ce;
}

.assistant-message-icon svg {
width: 1rem;
height: 1rem;
}

.assistant-search {
font-size: 0.875rem;
line-height: 1.25rem;
}

.assistant-search-icon {
display: flex;
justify-content: center;
align-items: center;
width: 1.5rem;
height: 1.5rem;
}

.assistant-search-icon svg {
width: 1rem;
height: 1rem;
}

.assistant-search-content {
align-content: center;
}

.assistant-search-phrase {
font-weight: 600;
}

/* Default styling for markdown-formatted assistant messages */
::deep ul {
list-style-type: disc;
margin-left: 1.5rem;
}

::deep ol {
list-style-type: decimal;
margin-left: 1.5rem;
}

::deep li {
margin: 0.5rem 0;
}

::deep strong {
font-weight: 600;
}

::deep h3 {
margin: 1rem 0;
font-weight: 600;
}

::deep p + p {
margin-top: 1rem;
}

::deep table {
margin: 1rem 0;
}

::deep th {
text-align: left;
border-bottom: 1px solid silver;
}

::deep th, ::deep td {
padding: 0.1rem 0.5rem;
}

::deep th, ::deep tr:nth-child(even) {
background-color: rgba(0, 0, 0, 0.05);
}
63 changes: 9 additions & 54 deletions ...tWithCustomData/ChatWithCustomData.Web-CSharp/Components/Pages/Chat/ChatMessageItem.razor
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,78 +12,33 @@ else if (Message.Role == ChatRole.Assistant)
{
foreach (var content in Message.Contents)
{
if (content is TextContent { Text: { Length: > 0 } text })
{
<div class="assistant-message">
<div>
<div class="assistant-message-icon">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor">
<path stroke-linecap="round" stroke-linejoin="round" d="M12 18v-5.25m0 0a6.01 6.01 0 0 0 1.5-.189m-1.5.189a6.01 6.01 0 0 1-1.5-.189m3.75 7.478a12.06 12.06 0 0 1-4.5 0m3.75 2.383a14.406 14.406 0 0 1-3 0M14.25 18v-.192c0-.983.658-1.823 1.508-2.316a7.5 7.5 0 1 0-7.517 0c.85.493 1.509 1.333 1.509 2.316V18" />
</svg>
</div>
</div>
<div class="assistant-message-header">Assistant</div>
<div class="assistant-message-text">
<assistant-message markdown="@text"></assistant-message>

@foreach (var citation in citations ?? [])
{
<ChatCitation File="@citation.File" PageNumber="@citation.Page" Quote="@citation.Quote" />
}
</div>
</div>
}
else if (content is FunctionCallContent { Name: "Search" } fcc && fcc.Arguments?.TryGetValue("searchPhrase", out var searchPhrase) is true)
{
<div class="assistant-search">
<div class="assistant-search-icon">
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor">
<path stroke-linecap="round" stroke-linejoin="round" d="m21 21-5.197-5.197m0 0A7.5 7.5 0 1 0 5.196 5.196a7.5 7.5 0 0 0 10.607 10.607Z" />
</svg>
</div>
<div class="assistant-search-content">
Searching:
<span class="assistant-search-phrase">@searchPhrase</span>
@if (fcc.Arguments?.TryGetValue("filenameFilter", out var filenameObj) is true && filenameObj is string filename && !string.IsNullOrEmpty(filename))
{
<text> in </text><span class="assistant-search-phrase">@filename</span>
}
</div>
</div>
}
<ChatAssistantContentItem Content="@content">
<TextContentFooter>
@foreach (var citation in citations ?? [])
{
<ChatCitation File="@citation.File" PageNumber="@citation.Page" Quote="@citation.Quote" />
}
</TextContentFooter>
</ChatAssistantContentItem>
}
}

@code {
private static readonly ConditionalWeakTable<ChatMessage, ChatMessageItem> SubscribersLookup = new();
private static readonly Regex CitationRegex = new(@"<citation filename='(?<file>[^']*)' page_number='(?<page>\d*)'>(?<quote>.*?)</citation>", RegexOptions.NonBacktracking);

private List<(string File, int? Page, string Quote)>? citations;

[Parameter, EditorRequired]
public required ChatMessage Message { get; set; }

[Parameter]
public bool InProgress { get; set;}

protected override void OnInitialized()
{
SubscribersLookup.AddOrUpdate(Message, this);

if (!InProgress && Message.Role == ChatRole.Assistant && Message.Text is { Length: > 0 } text)
if (Message.Role == ChatRole.Assistant && Message.Text is { Length: > 0 } text)
{
ParseCitations(text);
}
}

public static void NotifyChanged(ChatMessage source)
{
if (SubscribersLookup.TryGetValue(source, out var subscriber))
{
subscriber.StateHasChanged();
}
}

private void ParseCitations(string text)
{
var matches = CitationRegex.Matches(text);
Expand Down
Loading
Loading