[C#] feat: Implemented basic changes needed to support streaming tool…

… calls (#2214) ## Linked issues closes: #2098 ## Details Port the streaming support for tools augmentation to C#. #### Change details - Added StreamingChatToolCallsBuilder and SequenceBuilder helper classes. - Updated OpenAIModel class to process tool call chunks and return them as a message with ActionCalls. - Updated the LLMClient class to pass through the ActionCalls without closing the streaming response session. **code snippets**: **screenshots**: ## Attestation Checklist - [ ] My code follows the style guidelines of this project - I have checked for/fixed spelling, linting, and other errors - I have commented my code for clarity - I have made corresponding changes to the documentation (updating the doc strings in the code is sufficient) - My changes generate no new warnings - I have added tests that validates my changes, and provides sufficient test coverage. I have tested with: - Local testing - E2E testing in Teams - New and existing unit tests pass locally with my changes ### Additional information > Feel free to add other relevant information below --------- Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: kavin <[email protected]> Co-authored-by: Lily Du <[email protected]> Co-authored-by: lilydu <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Corina <[email protected]> Co-authored-by: Yiqing Zhao <[email protected]> Co-authored-by: Yiqing Zhao <[email protected]> Co-authored-by: Alex Acebo <[email protected]>
microsoft · Dec 9, 2024 · 2ba12e3 · 2ba12e3
1 parent 1c7d9c9
commit 2ba12e3
Show file tree

Hide file tree

Showing 66 changed files with 2,537 additions and 1,693 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -2,15 +2,15 @@
 
 # JS
 
-/js @aacebo @corinagum @lilyydu @singhk97
+/js @aacebo @corinagum @lilyydu @singhk97 @rajan-chari
 
 # .NET
 
-/dotnet @aacebo @corinagum @lilyydu @singhk97
+/dotnet @aacebo @corinagum @lilyydu @singhk97 @rajan-chari
 
 # Python
 
-/python @aacebo @corinagum @lilyydu @singhk97
+/python @aacebo @corinagum @lilyydu @singhk97 @rajan-chari
 
 # TTK
 

diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -71,7 +71,7 @@ jobs:
       - name: Test
         run: dotnet test Microsoft.TeamsAI.Tests/Microsoft.Teams.AI.Tests.csproj --verbosity normal --logger trx --results-directory ./TestResults --collect:"XPlat Code Coverage" --configuration Release
       - name: Coverage
-        uses: danielpalme/ReportGenerator-GitHub-Action@62f9e70ab348d56eee76d446b4db903a85ab0ea8 # 5.3.11
+        uses: danielpalme/ReportGenerator-GitHub-Action@810356ce07a94200154301fb73d878e327b2dd58 # 5.4.1
         with:
           reports: ${{ env.SOLUTION_DIR }}TestResults/*/coverage.cobertura.xml
           targetdir: ${{ env.SOLUTION_DIR }}TestResults/coverage

diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml
@@ -19,4 +19,4 @@ jobs:
       - name: 'Checkout Repository'
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@4081bf99e2866ebe428fc0477b69eb4fcda7220a # v4.4.0
+        uses: actions/dependency-review-action@3b139cfc5fae8b618d3eae3675e383bb1769c019 # v4.5.0
diff --git a/.github/workflows/dotnet-build-test-lint.yml b/.github/workflows/dotnet-build-test-lint.yml
@@ -43,7 +43,7 @@ jobs:
       - name: Test
         run: dotnet test Microsoft.TeamsAI.Tests/Microsoft.Teams.AI.Tests.csproj --no-restore --verbosity normal --logger trx --results-directory ./TestResults --collect:"XPlat Code Coverage" --configuration Release
       - name: Coverage
-        uses: danielpalme/ReportGenerator-GitHub-Action@62f9e70ab348d56eee76d446b4db903a85ab0ea8 # 5.3.11
+        uses: danielpalme/ReportGenerator-GitHub-Action@810356ce07a94200154301fb73d878e327b2dd58 # 5.4.1
         with:
           reports: ${{ env.SOLUTION_DIR }}TestResults/*/coverage.cobertura.xml
           targetdir: ${{ env.SOLUTION_DIR }}TestResults/coverage
@@ -55,6 +55,6 @@ jobs:
           name: testresults-dotnet-${{ matrix.dotnet-version }}
           path: ${{ env.SOLUTION_DIR }}TestResults
       - name: Harden Runner
-        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
         with:
           egress-policy: audit
diff --git a/.github/workflows/dotnet-codeql.yml b/.github/workflows/dotnet-codeql.yml
@@ -39,7 +39,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0
+        uses: github/codeql-action/init@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
         with:
           languages: csharp
       - name: Setup .NET
@@ -50,6 +50,6 @@ jobs:
         working-directory: dotnet/packages/Microsoft.TeamsAI/
         run: dotnet build Microsoft.Teams.AI.sln --configuration Release
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0
+        uses: github/codeql-action/analyze@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
         with:
           category: "/language:csharp"
diff --git a/.github/workflows/js-build-test-lint.yml b/.github/workflows/js-build-test-lint.yml
@@ -42,6 +42,6 @@ jobs:
       - name: Lint
         run: yarn lint
       - name: Harden Runner
-        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
         with:
           egress-policy: audit
diff --git a/.github/workflows/js-codeql.yml b/.github/workflows/js-codeql.yml
@@ -38,10 +38,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0
+        uses: github/codeql-action/init@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
         with:
           languages: javascript
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0
+        uses: github/codeql-action/analyze@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
         with:
           category: "/language:javascript"
diff --git a/.github/workflows/python-build-test-lint.yml b/.github/workflows/python-build-test-lint.yml
@@ -51,6 +51,6 @@ jobs:
         run: |
           python scripts/lint.py
       - name: Harden Runner
-        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
         with:
           egress-policy: audit
diff --git a/.github/workflows/python-codeql.yml b/.github/workflows/python-codeql.yml
@@ -38,10 +38,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0
+        uses: github/codeql-action/init@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
         with:
           languages: python
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0
+        uses: github/codeql-action/analyze@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
         with:
           category: "/language:python"
diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
@@ -66,6 +66,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0
+        uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
         with:
           sarif_file: results.sarif
diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Clients/LLMClient.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Clients/LLMClient.cs
@@ -1,4 +1,5 @@
-using Microsoft.Bot.Builder;
+using Google.Protobuf.WellKnownTypes;
+using Microsoft.Bot.Builder;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Teams.AI.AI.Models;
@@ -155,7 +156,6 @@ public async Task<PromptResponse> CompletePromptAsync(
         )
         {
             // Define event handlers
-            bool isStreaming = false;
             StreamingResponse? streamer = null;
 
             BeforeCompletionHandler handleBeforeCompletion = new((object sender, BeforeCompletionEventArgs args) =>
@@ -168,22 +168,26 @@ public async Task<PromptResponse> CompletePromptAsync(
 
                 if (args.Streaming)
                 {
-                    isStreaming = true;
-
-                    // Create streamer and send initial message
-                    streamer = new StreamingResponse(context);
-                    memory.SetValue("temp.streamer", streamer);
-
-                    if (this._enableFeedbackLoop != null)
+                    // Attach to any existing streamer
+                    // - see tool call note below to understand.
+                    streamer = (StreamingResponse?)memory.GetValue("temp.streamer");
+                    if (streamer == null)
                     {
-                        streamer.EnableFeedbackLoop = this._enableFeedbackLoop;
-                    }
+                        // Create streamer and send initial message
+                        streamer = new StreamingResponse(context);
+                        memory.SetValue("temp.streamer", streamer);
 
-                    streamer.EnableGeneratedByAILabel = true;
+                        if (this._enableFeedbackLoop != null)
+                        {
+                            streamer.EnableFeedbackLoop = this._enableFeedbackLoop;
+                        }
 
-                    if (!string.IsNullOrEmpty(this._startStreamingMessage))
-                    {
-                        streamer.QueueInformativeUpdate(this._startStreamingMessage!);
+                        streamer.EnableGeneratedByAILabel = true;
+
+                        if (!string.IsNullOrEmpty(this._startStreamingMessage))
+                        {
+                            streamer.QueueInformativeUpdate(this._startStreamingMessage!);
+                        }
                     }
                 }
             });
@@ -195,6 +199,15 @@ public async Task<PromptResponse> CompletePromptAsync(
                     return;
                 }
 
+
+                // Ignore content without text
+                // - The chunk is likely for a Tool Call.
+                // - See the tool call note below to understand why we're ignoring them.
+                if (args.Chunk.delta?.GetContent<string>() == null)
+                {
+                    return;
+                }
+
                 // Send chunk to client
                 string text = args.Chunk.delta?.GetContent<string>() ?? "";
                 IList<Citation>? citations = args.Chunk.delta?.Context?.Citations ?? null;
@@ -234,23 +247,32 @@ public async Task<PromptResponse> CompletePromptAsync(
                     cancellationToken
                 );
 
-                if (response.Status != PromptResponseStatus.Success)
-                {
-                    return response;
-                }
-                else
+                // Handle streaming responses
+                if (streamer != null)
                 {
-                    if (isStreaming)
+                    // Tool call handling
+                    // - We need to keep the streamer around during tool calls so we're just letting them return as normal
+                    //   messages minus the message content. The text content is being streamed to the client in chunks.
+                    // - When the tool call completes we'll call back into ActionPlanner and end up re-attaching to the
+                    //   streamer. This will result in us continuing to stream the response to the client. 
+                    if (response.Message?.ActionCalls != null && response.Message.ActionCalls.Count > 0)
                     {
-                        // Delete the message from the response to avoid sending it twice.
-                        response.Message = null;
+                        // Ensure content is empty for tool calls
+                        response.Message.Content = "";
                     }
-                }
+                    else
+                    {
+                        if (response.Status == PromptResponseStatus.Success)
+                        {
+                            // Delete message from response to avoid sending it twice
+                            response.Message = null;
+                        }
 
-                // End the stream
-                if (streamer != null)
-                {
-                    await streamer.EndStream();
+                        // End the stream and remove pointer from memory
+                        // - We're not listening for the response received event because we can't await the completion of events.
+                        await streamer.EndStream();
+                        memory.DeleteValue("temp.streamer");
+                    }
                 }
 
                 // Get input message/s

diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/OpenAIModel.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/OpenAIModel.cs
@@ -21,6 +21,7 @@
 using Microsoft.Teams.AI.Application;
 using System.Reflection;
 using System.Runtime.CompilerServices;
+using Microsoft.Extensions.Options;
 
 [assembly: InternalsVisibleTo("Microsoft.Teams.AI.Tests")]
 #pragma warning disable AOAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
@@ -214,7 +215,6 @@ public async Task<PromptResponse> CompletePromptAsync(ITurnContext turnContext,
 
             ChatCompletionOptions chatCompletionOptions = new()
             {
-                MaxOutputTokenCount = completion.MaxTokens,
                 Temperature = (float)completion.Temperature,
                 TopP = (float)completion.TopP,
                 PresencePenalty = (float)completion.PresencePenalty,
@@ -223,6 +223,7 @@ public async Task<PromptResponse> CompletePromptAsync(ITurnContext turnContext,
 
             if (isO1Model)
             {
+                chatCompletionOptions.MaxOutputTokenCount = completion.MaxTokens;
                 chatCompletionOptions.Temperature = 1;
                 chatCompletionOptions.TopP = 1;
                 chatCompletionOptions.PresencePenalty = 0;
@@ -282,6 +283,7 @@ public async Task<PromptResponse> CompletePromptAsync(ITurnContext turnContext,
                     };
                     AsyncCollectionResult<StreamingChatCompletionUpdate> streamCompletion = _openAIClient.GetChatClient(_deploymentName).CompleteChatStreamingAsync(chatMessages, chatCompletionOptions, cancellationToken);
 
+                    var toolCallBuilder = new StreamingChatToolCallsBuilder();
                     await foreach (StreamingChatCompletionUpdate delta in streamCompletion)
                     {
                         if (delta.Role != null)
@@ -295,9 +297,19 @@ public async Task<PromptResponse> CompletePromptAsync(ITurnContext turnContext,
                             message.Content += delta.ContentUpdate[0].Text;
                         }
 
-                        // TODO: Handle tool calls
+                        // Handle tool calls
+                        if (isToolsAugmentation && delta.ToolCallUpdates != null && delta.ToolCallUpdates.Count > 0)
+                        {
+                            foreach (var toolCallUpdate in delta.ToolCallUpdates)
+                            {
+                                toolCallBuilder.Append(toolCallUpdate);
+                            }
+                        }
 
-                        ChatMessage currDeltaMessage = new(delta);
+                        ChatMessage currDeltaMessage = new(delta)
+                        {
+                            ActionCalls = message.ActionCalls // Ensure ActionCalls are included
+                        };
                         PromptChunk chunk = new()
                         {
                             delta = currDeltaMessage
@@ -311,7 +323,19 @@ public async Task<PromptResponse> CompletePromptAsync(ITurnContext turnContext,
                             _logger.LogTrace("CHUNK", delta);
                         }
 
-                       Events!.OnChunkReceived(args);
+                        Events!.OnChunkReceived(args);
+                    }
+
+                    // Add any tool calls to message
+                    var toolCalls = toolCallBuilder.Build();
+                    if (toolCalls.Count > 0)
+                    {
+                        message.ActionCalls = new List<ActionCall>();
+                        foreach (var toolCall in toolCalls)
+                        {
+                            var actionCall = new ActionCall(toolCall);
+                            message.ActionCalls.Add(actionCall);
+                        }
                     }
 
                     promptResponse.Message = message;

diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/SequenceBuilder.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/SequenceBuilder.cs
@@ -0,0 +1,65 @@
+using System.Buffers;
+using System.Diagnostics;
+
+namespace Microsoft.Teams.AI.AI.Models
+{
+    public class SequenceBuilder<T>
+    {
+        private Segment _first;
+        private Segment _last;
+
+        public void Append(ReadOnlyMemory<T> data)
+        {
+            if (_first == null)
+            {
+                Debug.Assert(_last == null);
+                _first = new Segment(data);
+                _last = _first;
+            }
+            else
+            {
+                _last = _last!.Append(data);
+            }
+        }
+
+        public ReadOnlySequence<T> Build()
+        {
+            if (_first == null)
+            {
+                Debug.Assert(_last == null);
+                return ReadOnlySequence<T>.Empty;
+            }
+
+            if (_first == _last)
+            {
+                Debug.Assert(_first.Next == null);
+                return new ReadOnlySequence<T>(_first.Memory);
+            }
+
+            return new ReadOnlySequence<T>(_first, 0, _last!, _last!.Memory.Length);
+        }
+
+        private sealed class Segment : ReadOnlySequenceSegment<T>
+        {
+            public Segment(ReadOnlyMemory<T> items) : this(items, 0)
+            {
+            }
+
+            private Segment(ReadOnlyMemory<T> items, long runningIndex)
+            {
+                Debug.Assert(runningIndex >= 0);
+                Memory = items;
+                RunningIndex = runningIndex;
+            }
+
+            public Segment Append(ReadOnlyMemory<T> items)
+            {
+                long runningIndex;
+                checked { runningIndex = RunningIndex + Memory.Length; }
+                Segment segment = new(items, runningIndex);
+                Next = segment;
+                return segment;
+            }
+        }
+    }
+}