Merge branch 'main' into lilyydu/stream-citations-python

.github/CODEOWNERS

@@ -2,15 +2,15 @@
 
 # JS
 
-/js @aacebo @corinagum @lilyydu @singhk97 @rajan-chari
+/js @aacebo @corinagum @lilyydu @singhk97 @rajan-chari @heyitsaamir
 
 # .NET
 
-/dotnet @aacebo @corinagum @lilyydu @singhk97 @rajan-chari
+/dotnet @aacebo @corinagum @lilyydu @singhk97 @rajan-chari @heyitsaamir
 
 # Python
 
-/python @aacebo @corinagum @lilyydu @singhk97 @rajan-chari
+/python @aacebo @corinagum @lilyydu @singhk97 @rajan-chari @heyitsaamir
 
 # TTK
 

.github/workflows/dependency-review.yml

@@ -19,4 +19,4 @@ jobs:
       - name: 'Checkout Repository'
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@4081bf99e2866ebe428fc0477b69eb4fcda7220a # v4.4.0
+        uses: actions/dependency-review-action@3b139cfc5fae8b618d3eae3675e383bb1769c019 # v4.5.0

.github/workflows/dotnet-build-test-lint.yml

@@ -55,6 +55,6 @@ jobs:
           name: testresults-dotnet-${{ matrix.dotnet-version }}
           path: ${{ env.SOLUTION_DIR }}TestResults
       - name: Harden Runner
-        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
         with:
           egress-policy: audit

.github/workflows/dotnet-codeql.yml

@@ -39,7 +39,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
+        uses: github/codeql-action/init@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9
         with:
           languages: csharp
       - name: Setup .NET
@@ -50,6 +50,6 @@ jobs:
         working-directory: dotnet/packages/Microsoft.TeamsAI/
         run: dotnet build Microsoft.Teams.AI.sln --configuration Release
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
+        uses: github/codeql-action/analyze@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9
         with:
           category: "/language:csharp"

.github/workflows/js-build-test-lint.yml

@@ -42,6 +42,6 @@ jobs:
       - name: Lint
         run: yarn lint
       - name: Harden Runner
-        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
         with:
           egress-policy: audit

.github/workflows/js-codeql.yml

@@ -38,10 +38,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
+        uses: github/codeql-action/init@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9
         with:
           languages: javascript
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
+        uses: github/codeql-action/analyze@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9
         with:
           category: "/language:javascript"

.github/workflows/python-build-test-lint.yml

@@ -51,6 +51,6 @@ jobs:
         run: |
           python scripts/lint.py
       - name: Harden Runner
-        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
+        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
         with:
           egress-policy: audit

.github/workflows/python-codeql.yml

@@ -38,10 +38,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
+        uses: github/codeql-action/init@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9
         with:
           languages: python
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
+        uses: github/codeql-action/analyze@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9
         with:
           category: "/language:python"

.github/workflows/scorecards.yml

@@ -66,6 +66,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
+        uses: github/codeql-action/upload-sarif@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9
         with:
           sarif_file: results.sarif

dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI.Tests/AITests/LLMClientTests.cs

@@ -99,8 +99,10 @@ public async Task Test_CompletePromptAsync_PromptResponse_NotSuccess()
             LLMClientOptions<object> options = new(promptCompletionModel, promptTemplate) { MaxHistoryMessages = 1 };
             LLMClient<object> client = new(options, null);
             TestMemory memory = new();
+            ChatMessage message = new ChatMessage("Hi there");
             promptCompletionModel.Results.Enqueue(new()
             {
+                Input = new List<ChatMessage>() { message },
                 Status = PromptResponseStatus.Error,
                 Error = new TeamsAIException("test")
             });
@@ -113,7 +115,10 @@ public async Task Test_CompletePromptAsync_PromptResponse_NotSuccess()
             Assert.Equal(PromptResponseStatus.Error, response.Status);
             Assert.NotNull(response.Error);
             Assert.Equal("test", response.Error.Message);
-            Assert.Empty(memory.Values);
+            Assert.Single(memory.Values);
+
+            IList<ChatMessage> conversation_history = (IList<ChatMessage>)memory.GetValue("conversation.history")!;
+            Assert.True(conversation_history[0].Content  ==  message.Content);
         }
 
         [Fact]

dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/.editorconfig

@@ -2,3 +2,6 @@
 
 # CS0618: Type or member is obsolete
 dotnet_diagnostic.CS0618.severity = silent
+
+# IDE0028: Simplify collection initialization
+dotnet_diagnostic.IDE0028.severity = silent

dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Action/AIEntity.cs

@@ -94,7 +94,7 @@ public class ClientCitationAppearance
         public string Name { get; set; } = string.Empty;
 
         /// <summary>
-        /// Optional. The citation appreance text. It is ignored in Teams.
+        /// Optional. The citation appearance text.
         /// </summary>
         [JsonProperty(PropertyName = "text")]
         public string? Text { get; set; }
@@ -112,10 +112,10 @@ public class ClientCitationAppearance
         public string Abstract { get; set; } = string.Empty;
 
         /// <summary>
-        /// The encoding format used for the icon.
+        /// Optional. Encoding format of the `citation.appearance.text` field.
         /// </summary>
         [JsonProperty(PropertyName = "encodingFormat")]
-        public string EncodingFormat { get; set; } = "text/html";
+        public string? EncodingFormat { get; set; }
 
         /// <summary>
         /// The icon provided in the citation ui.

dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Clients/LLMClient.cs

@@ -1,4 +1,5 @@
-using Microsoft.Bot.Builder;
+using Google.Protobuf.WellKnownTypes;
+using Microsoft.Bot.Builder;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Teams.AI.AI.Models;
@@ -155,7 +156,6 @@ public async Task<PromptResponse> CompletePromptAsync(
         )
         {
             // Define event handlers
-            bool isStreaming = false;
             StreamingResponse? streamer = null;
 
             BeforeCompletionHandler handleBeforeCompletion = new((object sender, BeforeCompletionEventArgs args) =>
@@ -168,22 +168,26 @@ public async Task<PromptResponse> CompletePromptAsync(
 
                 if (args.Streaming)
                 {
-                    isStreaming = true;
-
-                    // Create streamer and send initial message
-                    streamer = new StreamingResponse(context);
-                    memory.SetValue("temp.streamer", streamer);
-
-                    if (this._enableFeedbackLoop != null)
+                    // Attach to any existing streamer
+                    // - see tool call note below to understand.
+                    streamer = (StreamingResponse?)memory.GetValue("temp.streamer");
+                    if (streamer == null)
                     {
-                        streamer.EnableFeedbackLoop = this._enableFeedbackLoop;
-                    }
+                        // Create streamer and send initial message
+                        streamer = new StreamingResponse(context);
+                        memory.SetValue("temp.streamer", streamer);
 
-                    streamer.EnableGeneratedByAILabel = true;
+                        if (this._enableFeedbackLoop != null)
+                        {
+                            streamer.EnableFeedbackLoop = this._enableFeedbackLoop;
+                        }
 
-                    if (!string.IsNullOrEmpty(this._startStreamingMessage))
-                    {
-                        streamer.QueueInformativeUpdate(this._startStreamingMessage!);
+                        streamer.EnableGeneratedByAILabel = true;
+
+                        if (!string.IsNullOrEmpty(this._startStreamingMessage))
+                        {
+                            streamer.QueueInformativeUpdate(this._startStreamingMessage!);
+                        }
                     }
                 }
             });
@@ -195,6 +199,15 @@ public async Task<PromptResponse> CompletePromptAsync(
                     return;
                 }
 
+
+                // Ignore content without text
+                // - The chunk is likely for a Tool Call.
+                // - See the tool call note below to understand why we're ignoring them.
+                if (args.Chunk.delta?.GetContent<string>() == null)
+                {
+                    return;
+                }
+
                 // Send chunk to client
                 string text = args.Chunk.delta?.GetContent<string>() ?? "";
                 IList<Citation>? citations = args.Chunk.delta?.Context?.Citations ?? null;
@@ -234,23 +247,32 @@ public async Task<PromptResponse> CompletePromptAsync(
                     cancellationToken
                 );
 
-                if (response.Status != PromptResponseStatus.Success)
-                {
-                    return response;
-                }
-                else
+                // Handle streaming responses
+                if (streamer != null)
                 {
-                    if (isStreaming)
+                    // Tool call handling
+                    // - We need to keep the streamer around during tool calls so we're just letting them return as normal
+                    //   messages minus the message content. The text content is being streamed to the client in chunks.
+                    // - When the tool call completes we'll call back into ActionPlanner and end up re-attaching to the
+                    //   streamer. This will result in us continuing to stream the response to the client. 
+                    if (response.Message?.ActionCalls != null && response.Message.ActionCalls.Count > 0)
                     {
-                        // Delete the message from the response to avoid sending it twice.
-                        response.Message = null;
+                        // Ensure content is empty for tool calls
+                        response.Message.Content = "";
                     }
-                }
+                    else
+                    {
+                        if (response.Status == PromptResponseStatus.Success)
+                        {
+                            // Delete message from response to avoid sending it twice
+                            response.Message = null;
+                        }
 
-                // End the stream
-                if (streamer != null)
-                {
-                    await streamer.EndStream();
+                        // End the stream and remove pointer from memory
+                        // - We're not listening for the response received event because we can't await the completion of events.
+                        await streamer.EndStream();
+                        memory.DeleteValue("temp.streamer");
+                    }
                 }
 
                 // Get input message/s

dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/OpenAIModel.cs

@@ -21,6 +21,7 @@
 using Microsoft.Teams.AI.Application;
 using System.Reflection;
 using System.Runtime.CompilerServices;
+using Microsoft.Extensions.Options;
 
 [assembly: InternalsVisibleTo("Microsoft.Teams.AI.Tests")]
 #pragma warning disable AOAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
@@ -214,7 +215,6 @@ public async Task<PromptResponse> CompletePromptAsync(ITurnContext turnContext,
 
             ChatCompletionOptions chatCompletionOptions = new()
             {
-                MaxOutputTokenCount = completion.MaxTokens,
                 Temperature = (float)completion.Temperature,
                 TopP = (float)completion.TopP,
                 PresencePenalty = (float)completion.PresencePenalty,
@@ -223,6 +223,7 @@ public async Task<PromptResponse> CompletePromptAsync(ITurnContext turnContext,
 
             if (isO1Model)
             {
+                chatCompletionOptions.MaxOutputTokenCount = completion.MaxTokens;
                 chatCompletionOptions.Temperature = 1;
                 chatCompletionOptions.TopP = 1;
                 chatCompletionOptions.PresencePenalty = 0;
@@ -282,6 +283,7 @@ public async Task<PromptResponse> CompletePromptAsync(ITurnContext turnContext,
                     };
                     AsyncCollectionResult<StreamingChatCompletionUpdate> streamCompletion = _openAIClient.GetChatClient(_deploymentName).CompleteChatStreamingAsync(chatMessages, chatCompletionOptions, cancellationToken);
 
+                    var toolCallBuilder = new StreamingChatToolCallsBuilder();
                     await foreach (StreamingChatCompletionUpdate delta in streamCompletion)
                     {
                         if (delta.Role != null)
@@ -295,9 +297,19 @@ public async Task<PromptResponse> CompletePromptAsync(ITurnContext turnContext,
                             message.Content += delta.ContentUpdate[0].Text;
                         }
 
-                        // TODO: Handle tool calls
+                        // Handle tool calls
+                        if (isToolsAugmentation && delta.ToolCallUpdates != null && delta.ToolCallUpdates.Count > 0)
+                        {
+                            foreach (var toolCallUpdate in delta.ToolCallUpdates)
+                            {
+                                toolCallBuilder.Append(toolCallUpdate);
+                            }
+                        }
 
-                        ChatMessage currDeltaMessage = new(delta);
+                        ChatMessage currDeltaMessage = new(delta)
+                        {
+                            ActionCalls = message.ActionCalls // Ensure ActionCalls are included
+                        };
                         PromptChunk chunk = new()
                         {
                             delta = currDeltaMessage
@@ -311,7 +323,19 @@ public async Task<PromptResponse> CompletePromptAsync(ITurnContext turnContext,
                             _logger.LogTrace("CHUNK", delta);
                         }
 
-                       Events!.OnChunkReceived(args);
+                        Events!.OnChunkReceived(args);
+                    }
+
+                    // Add any tool calls to message
+                    var toolCalls = toolCallBuilder.Build();
+                    if (toolCalls.Count > 0)
+                    {
+                        message.ActionCalls = new List<ActionCall>();
+                        foreach (var toolCall in toolCalls)
+                        {
+                            var actionCall = new ActionCall(toolCall);
+                            message.ActionCalls.Add(actionCall);
+                        }
                     }
 
                     promptResponse.Message = message;