[C#] feat: Implemented basic changes needed to support streaming tool calls

.github/CODEOWNERS

@@ -2,15 +2,15 @@
 
 # JS
 
-/js @aacebo @corinagum @lilyydu @singhk97
+/js @aacebo @corinagum @lilyydu @singhk97 @rajan-chari
 
 # .NET
 
-/dotnet @aacebo @corinagum @lilyydu @singhk97
+/dotnet @aacebo @corinagum @lilyydu @singhk97 @rajan-chari
 
 # Python
 
-/python @aacebo @corinagum @lilyydu @singhk97
+/python @aacebo @corinagum @lilyydu @singhk97 @rajan-chari
 
 # TTK
 

.github/workflows/coverage.yml

@@ -71,7 +71,7 @@ jobs:
       - name: Test
         run: dotnet test Microsoft.TeamsAI.Tests/Microsoft.Teams.AI.Tests.csproj --verbosity normal --logger trx --results-directory ./TestResults --collect:"XPlat Code Coverage" --configuration Release
       - name: Coverage
-        uses: danielpalme/ReportGenerator-GitHub-Action@62f9e70ab348d56eee76d446b4db903a85ab0ea8 # 5.3.11
+        uses: danielpalme/ReportGenerator-GitHub-Action@810356ce07a94200154301fb73d878e327b2dd58 # 5.4.1
         with:
           reports: ${{ env.SOLUTION_DIR }}TestResults/*/coverage.cobertura.xml
           targetdir: ${{ env.SOLUTION_DIR }}TestResults/coverage

.github/workflows/dotnet-build-test-lint.yml

@@ -43,7 +43,7 @@ jobs:
       - name: Test
         run: dotnet test Microsoft.TeamsAI.Tests/Microsoft.Teams.AI.Tests.csproj --no-restore --verbosity normal --logger trx --results-directory ./TestResults --collect:"XPlat Code Coverage" --configuration Release
       - name: Coverage
-        uses: danielpalme/ReportGenerator-GitHub-Action@62f9e70ab348d56eee76d446b4db903a85ab0ea8 # 5.3.11
+        uses: danielpalme/ReportGenerator-GitHub-Action@810356ce07a94200154301fb73d878e327b2dd58 # 5.4.1
         with:
           reports: ${{ env.SOLUTION_DIR }}TestResults/*/coverage.cobertura.xml
           targetdir: ${{ env.SOLUTION_DIR }}TestResults/coverage

.github/workflows/dotnet-codeql.yml

@@ -39,7 +39,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0
+        uses: github/codeql-action/init@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
         with:
           languages: csharp
       - name: Setup .NET
@@ -50,6 +50,6 @@ jobs:
         working-directory: dotnet/packages/Microsoft.TeamsAI/
         run: dotnet build Microsoft.Teams.AI.sln --configuration Release
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0
+        uses: github/codeql-action/analyze@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
         with:
           category: "/language:csharp"

.github/workflows/js-codeql.yml

@@ -38,10 +38,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0
+        uses: github/codeql-action/init@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
         with:
           languages: javascript
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0
+        uses: github/codeql-action/analyze@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
         with:
           category: "/language:javascript"

.github/workflows/python-codeql.yml

@@ -38,10 +38,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0
+        uses: github/codeql-action/init@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
         with:
           languages: python
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0
+        uses: github/codeql-action/analyze@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
         with:
           category: "/language:python"

.github/workflows/scorecards.yml

@@ -66,6 +66,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0
+        uses: github/codeql-action/upload-sarif@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
         with:
           sarif_file: results.sarif

dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Clients/LLMClient.cs

@@ -1,4 +1,5 @@
-using Microsoft.Bot.Builder;
+using Google.Protobuf.WellKnownTypes;
+using Microsoft.Bot.Builder;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Teams.AI.AI.Models;
@@ -155,7 +156,6 @@ public async Task<PromptResponse> CompletePromptAsync(
         )
         {
             // Define event handlers
-            bool isStreaming = false;
             StreamingResponse? streamer = null;
 
             BeforeCompletionHandler handleBeforeCompletion = new((object sender, BeforeCompletionEventArgs args) =>
@@ -168,22 +168,26 @@ public async Task<PromptResponse> CompletePromptAsync(
 
                 if (args.Streaming)
                 {
-                    isStreaming = true;
-
-                    // Create streamer and send initial message
-                    streamer = new StreamingResponse(context);
-                    memory.SetValue("temp.streamer", streamer);
-
-                    if (this._enableFeedbackLoop != null)
+                    // Attach to any existing streamer
+                    // - see tool call note below to understand.
+                    streamer = (StreamingResponse?)memory.GetValue("temp.streamer");
+                    if (streamer == null)
                     {
-                        streamer.EnableFeedbackLoop = this._enableFeedbackLoop;
-                    }
+                        // Create streamer and send initial message
+                        streamer = new StreamingResponse(context);
+                        memory.SetValue("temp.streamer", streamer);
 
-                    streamer.EnableGeneratedByAILabel = true;
+                        if (this._enableFeedbackLoop != null)
+                        {
+                            streamer.EnableFeedbackLoop = this._enableFeedbackLoop;
+                        }
 
-                    if (!string.IsNullOrEmpty(this._startStreamingMessage))
-                    {
-                        streamer.QueueInformativeUpdate(this._startStreamingMessage!);
+                        streamer.EnableGeneratedByAILabel = true;
+
+                        if (!string.IsNullOrEmpty(this._startStreamingMessage))
+                        {
+                            streamer.QueueInformativeUpdate(this._startStreamingMessage!);
+                        }
                     }
                 }
             });
@@ -195,6 +199,15 @@ public async Task<PromptResponse> CompletePromptAsync(
                     return;
                 }
 
+
+                // Ignore content without text
+                // - The chunk is likely for a Tool Call.
+                // - See the tool call note below to understand why we're ignoring them.
+                if (args.Chunk.delta?.GetContent<string>() == null)
+                {
+                    return;
+                }
+
                 // Send chunk to client
                 string text = args.Chunk.delta?.GetContent<string>() ?? "";
                 IList<Citation>? citations = args.Chunk.delta?.Context?.Citations ?? null;
@@ -234,23 +247,32 @@ public async Task<PromptResponse> CompletePromptAsync(
                     cancellationToken
                 );
 
-                if (response.Status != PromptResponseStatus.Success)
-                {
-                    return response;
-                }
-                else
+                // Handle streaming responses
+                if (streamer != null)
                 {
-                    if (isStreaming)
+                    // Tool call handling
+                    // - We need to keep the streamer around during tool calls so we're just letting them return as normal
+                    //   messages minus the message content. The text content is being streamed to the client in chunks.
+                    // - When the tool call completes we'll call back into ActionPlanner and end up re-attaching to the
+                    //   streamer. This will result in us continuing to stream the response to the client. 
+                    if (response.Message?.ActionCalls != null && response.Message.ActionCalls.Count > 0)
                     {
-                        // Delete the message from the response to avoid sending it twice.
-                        response.Message = null;
+                        // Ensure content is empty for tool calls
+                        response.Message.Content = "";
                     }
-                }
+                    else
+                    {
+                        if (response.Status == PromptResponseStatus.Success)
+                        {
+                            // Delete message from response to avoid sending it twice
+                            response.Message = null;
+                        }
 
-                // End the stream
-                if (streamer != null)
-                {
-                    await streamer.EndStream();
+                        // End the stream and remove pointer from memory
+                        // - We're not listening for the response received event because we can't await the completion of events.
+                        await streamer.EndStream();
+                        memory.DeleteValue("temp.streamer");
+                    }
                 }
 
                 // Get input message/s

dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/OpenAIModel.cs

@@ -282,6 +282,7 @@ public async Task<PromptResponse> CompletePromptAsync(ITurnContext turnContext,
                     };
                     AsyncCollectionResult<StreamingChatCompletionUpdate> streamCompletion = _openAIClient.GetChatClient(_deploymentName).CompleteChatStreamingAsync(chatMessages, chatCompletionOptions, cancellationToken);
 
+                    var toolCallBuilder = new StreamingChatToolCallsBuilder();
                     await foreach (StreamingChatCompletionUpdate delta in streamCompletion)
                     {
                         if (delta.Role != null)
@@ -295,9 +296,19 @@ public async Task<PromptResponse> CompletePromptAsync(ITurnContext turnContext,
                             message.Content += delta.ContentUpdate[0].Text;
                         }
 
-                        // TODO: Handle tool calls
+                        // Handle tool calls
+                        if (isToolsAugmentation && delta.ToolCallUpdates != null && delta.ToolCallUpdates.Count > 0)
+                        {
+                            foreach (var toolCallUpdate in delta.ToolCallUpdates)
+                            {
+                                toolCallBuilder.Append(toolCallUpdate);
+                            }
+                        }
 
-                        ChatMessage currDeltaMessage = new(delta);
+                        ChatMessage currDeltaMessage = new(delta)
+                        {
+                            ActionCalls = message.ActionCalls // Ensure ActionCalls are included
+                        };
                         PromptChunk chunk = new()
                         {
                             delta = currDeltaMessage
@@ -311,7 +322,19 @@ public async Task<PromptResponse> CompletePromptAsync(ITurnContext turnContext,
                             _logger.LogTrace("CHUNK", delta);
                         }
 
-                       Events!.OnChunkReceived(args);
+                        Events!.OnChunkReceived(args);
+                    }
+
+                    // Add any tool calls to message
+                    var toolCalls = toolCallBuilder.Build();
+                    if (toolCalls.Count > 0)
+                    {
+                        message.ActionCalls = new List<ActionCall>();
+                        foreach (var toolCall in toolCalls)
+                        {
+                            var actionCall = new ActionCall(toolCall);
+                            message.ActionCalls.Add(actionCall);
+                        }
                     }
 
                     promptResponse.Message = message;

dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/SequenceBuilder.cs

@@ -0,0 +1,65 @@
+using System.Buffers;
+using System.Diagnostics;
+
+namespace Microsoft.Teams.AI.AI.Models
+{
+    public class SequenceBuilder<T>
+    {
+        private Segment _first;
+        private Segment _last;
+
+        public void Append(ReadOnlyMemory<T> data)
+        {
+            if (_first == null)
+            {
+                Debug.Assert(_last == null);
+                _first = new Segment(data);
+                _last = _first;
+            }
+            else
+            {
+                _last = _last!.Append(data);
+            }
+        }
+
+        public ReadOnlySequence<T> Build()
+        {
+            if (_first == null)
+            {
+                Debug.Assert(_last == null);
+                return ReadOnlySequence<T>.Empty;
+            }
+
+            if (_first == _last)
+            {
+                Debug.Assert(_first.Next == null);
+                return new ReadOnlySequence<T>(_first.Memory);
+            }
+
+            return new ReadOnlySequence<T>(_first, 0, _last!, _last!.Memory.Length);
+        }
+
+        private sealed class Segment : ReadOnlySequenceSegment<T>
+        {
+            public Segment(ReadOnlyMemory<T> items) : this(items, 0)
+            {
+            }
+
+            private Segment(ReadOnlyMemory<T> items, long runningIndex)
+            {
+                Debug.Assert(runningIndex >= 0);
+                Memory = items;
+                RunningIndex = runningIndex;
+            }
+
+            public Segment Append(ReadOnlyMemory<T> items)
+            {
+                long runningIndex;
+                checked { runningIndex = RunningIndex + Memory.Length; }
+                Segment segment = new(items, runningIndex);
+                Next = segment;
+                return segment;
+            }
+        }
+    }
+}

dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/StreamingChatToolCallsBuilder.cs

@@ -0,0 +1,59 @@
+using OpenAI.Chat;
+using System.Buffers;
+
+namespace Microsoft.Teams.AI.AI.Models
+{
+    public class StreamingChatToolCallsBuilder
+    {
+        private readonly Dictionary<int, string> _indexToToolCallId = [];
+        private readonly Dictionary<int, string> _indexToFunctionName = [];
+        private readonly Dictionary<int, SequenceBuilder<byte>> _indexToFunctionArguments = [];
+
+        public void Append(StreamingChatToolCallUpdate toolCallUpdate)
+        {
+            // Keep track of which tool call ID belongs to this update index.
+            if (toolCallUpdate.ToolCallId != null)
+            {
+                _indexToToolCallId[toolCallUpdate.Index] = toolCallUpdate.ToolCallId;
+            }
+
+            // Keep track of which function name belongs to this update index.
+            if (toolCallUpdate.FunctionName != null)
+            {
+                _indexToFunctionName[toolCallUpdate.Index] = toolCallUpdate.FunctionName;
+            }
+
+            // Keep track of which function arguments belong to this update index,
+            // and accumulate the arguments as new updates arrive.
+            if (toolCallUpdate.FunctionArgumentsUpdate != null && !toolCallUpdate.FunctionArgumentsUpdate.ToMemory().IsEmpty)
+            {
+                if (!_indexToFunctionArguments.TryGetValue(toolCallUpdate.Index, out SequenceBuilder<byte> argumentsBuilder))
+                {
+                    argumentsBuilder = new SequenceBuilder<byte>();
+                    _indexToFunctionArguments[toolCallUpdate.Index] = argumentsBuilder;
+                }
+
+                argumentsBuilder.Append(toolCallUpdate.FunctionArgumentsUpdate);
+            }
+        }
+
+        public IReadOnlyList<ChatToolCall> Build()
+        {
+            List<ChatToolCall> toolCalls = [];
+
+            foreach (KeyValuePair<int, string> indexToToolCallIdPair in _indexToToolCallId)
+            {
+                ReadOnlySequence<byte> sequence = _indexToFunctionArguments[indexToToolCallIdPair.Key].Build();
+
+                ChatToolCall toolCall = ChatToolCall.CreateFunctionToolCall(
+                    id: indexToToolCallIdPair.Value,
+                    functionName: _indexToFunctionName[indexToToolCallIdPair.Key],
+                    functionArguments: BinaryData.FromBytes(sequence.ToArray()));
+
+                toolCalls.Add(toolCall);
+            }
+
+            return toolCalls;
+        }
+    }
+}

dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/Application/StreamingResponse.cs

@@ -313,7 +313,7 @@ private async Task SendActivity(Activity activity)
                 {
                     Properties = JObject.FromObject(new {
                         streamId = ((StreamingChannelData) activity.ChannelData).streamId,
-                        streamType = ((StreamingChannelData) activity.ChannelData).StreamType,
+                        streamType = ((StreamingChannelData) activity.ChannelData).StreamType.ToString(),
                         streamSequence = ((StreamingChannelData) activity.ChannelData).StreamSequence,
 
                     })

dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/Microsoft.Teams.AI.csproj

@@ -7,7 +7,7 @@
     <Nullable>enable</Nullable>
     <PackageId>Microsoft.Teams.AI</PackageId>
     <Product>Microsoft Teams AI SDK</Product>
-    <Version>1.8.0</Version>
+    <Version>1.8.1</Version>
     <Authors>Microsoft</Authors>
     <Company>Microsoft</Company>
     <Copyright>© Microsoft Corporation. All rights reserved.</Copyright>