[C#] feat: change PredictedSayCommand to include response `ChatMess…

…age` (#1610) ## Linked issues closes: #minor ## Details update `PredictedSayCommand` to have a `ChatMessage` response instead of a `string` type so that we can access citations and other message data from action handlers.
microsoft · May 7, 2024 · e3beb4d · e3beb4d
1 parent 7ab3f76
commit e3beb4d
Show file tree

Hide file tree

Showing 13 changed files with 115 additions and 23 deletions.
diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI.Tests/AITests/AITests.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI.Tests/AITests/AITests.cs
@@ -197,7 +197,7 @@ public string DoCommand([ActionName] string action)
             [Action(AIConstants.SayCommandActionName)]
             public string SayCommand([ActionParameters] PredictedSayCommand command)
             {
-                SayActionRecord.Add(command.Response);
+                SayActionRecord.Add(command.Response.Content);
                 return string.Empty;
             }
         }

diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI.Tests/AITests/AssistantsPlannerTests.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI.Tests/AITests/AssistantsPlannerTests.cs
@@ -42,7 +42,7 @@ public async Task Test_BeginTaskAsync_Assistant_Single_Reply()
             Assert.NotNull(plan.Commands);
             Assert.Single(plan.Commands);
             Assert.Equal(AIConstants.SayCommand, plan.Commands[0].Type);
-            Assert.Equal("welcome", ((PredictedSayCommand)plan.Commands[0]).Response);
+            Assert.Equal("welcome", ((PredictedSayCommand)plan.Commands[0]).Response.Content);
         }
 
         [Fact]
@@ -74,7 +74,7 @@ public async Task Test_BeginTaskAsync_Assistant_WaitForCurrentRun()
             Assert.NotNull(plan.Commands);
             Assert.Single(plan.Commands);
             Assert.Equal(AIConstants.SayCommand, plan.Commands[0].Type);
-            Assert.Equal("welcome", ((PredictedSayCommand)plan.Commands[0]).Response);
+            Assert.Equal("welcome", ((PredictedSayCommand)plan.Commands[0]).Response.Content);
         }
 
         [Fact]
@@ -110,7 +110,7 @@ public async Task Test_BeginTaskAsync_Assistant_WaitForPreviousRun()
             Assert.NotNull(plan.Commands);
             Assert.Single(plan.Commands);
             Assert.Equal(AIConstants.SayCommand, plan.Commands[0].Type);
-            Assert.Equal("welcome", ((PredictedSayCommand)plan.Commands[0]).Response);
+            Assert.Equal("welcome", ((PredictedSayCommand)plan.Commands[0]).Response.Content);
         }
 
         [Fact]
@@ -243,7 +243,7 @@ public async Task Test_ContinueTaskAsync_Assistant_RequiresAction()
             Assert.NotNull(plan2.Commands);
             Assert.Single(plan2.Commands);
             Assert.Equal(AIConstants.SayCommand, plan2.Commands[0].Type);
-            Assert.Equal("welcome", ((PredictedSayCommand)plan2.Commands[0]).Response);
+            Assert.Equal("welcome", ((PredictedSayCommand)plan2.Commands[0]).Response.Content);
             Assert.Single(turnState.SubmitToolMap);
             Assert.Equal("test-action", turnState.SubmitToolMap.First().Key);
             Assert.Equal("test-tool-id", turnState.SubmitToolMap.First().Value);
@@ -291,7 +291,7 @@ public async Task Test_ContinueTaskAsync_Assistant_IgnoreRedundantAction()
             Assert.NotNull(plan2.Commands);
             Assert.Single(plan2.Commands);
             Assert.Equal(AIConstants.SayCommand, plan2.Commands[0].Type);
-            Assert.Equal("welcome", ((PredictedSayCommand)plan2.Commands[0]).Response);
+            Assert.Equal("welcome", ((PredictedSayCommand)plan2.Commands[0]).Response.Content);
             Assert.Single(turnState.SubmitToolMap);
             Assert.Equal("test-action", turnState.SubmitToolMap.First().Key);
             Assert.Equal("test-tool-id", turnState.SubmitToolMap.First().Value);
@@ -328,9 +328,9 @@ public async Task Test_ContinueTaskAsync_Assistant_MultipleMessages()
             Assert.NotNull(plan.Commands);
             Assert.Equal(3, plan.Commands.Count);
             Assert.Equal(AIConstants.SayCommand, plan.Commands[0].Type);
-            Assert.Equal("welcome", ((PredictedSayCommand)plan.Commands[0]).Response);
-            Assert.Equal("message 1", ((PredictedSayCommand)plan.Commands[1]).Response);
-            Assert.Equal("message 2", ((PredictedSayCommand)plan.Commands[2]).Response);
+            Assert.Equal("welcome", ((PredictedSayCommand)plan.Commands[0]).Response.Content);
+            Assert.Equal("message 1", ((PredictedSayCommand)plan.Commands[1]).Response.Content);
+            Assert.Equal("message 2", ((PredictedSayCommand)plan.Commands[2]).Response.Content);
         }
 
         private static async Task<AssistantsState> _CreateAssistantsState()

diff --git a/...osoft.TeamsAI/Microsoft.TeamsAI.Tests/AITests/Augmentations/MonologueAugmentationTests.cs b/...osoft.TeamsAI/Microsoft.TeamsAI.Tests/AITests/Augmentations/MonologueAugmentationTests.cs
@@ -254,7 +254,15 @@ public async void Test_CreatePlanFromResponseAsync_SayCommand_ShouldSucceed()
                 Status = PromptResponseStatus.Success,
                 Message = new(ChatRole.Assistant)
                 {
-                    Content = JsonSerializer.Serialize(monologue)
+                    Content = JsonSerializer.Serialize(monologue),
+                    Context = new()
+                    {
+                        Intent = "test intent",
+                        Citations = new List<Citation>
+                        {
+                            new("content", "title", "url")
+                        }
+                    }
                 }
             };
 
@@ -267,7 +275,11 @@ public async void Test_CreatePlanFromResponseAsync_SayCommand_ShouldSucceed()
             Assert.NotNull(plan);
             Assert.Equal(1, plan.Commands.Count);
             Assert.Equal("SAY", plan.Commands[0].Type);
-            Assert.Equal("hello world", (plan.Commands[0] as PredictedSayCommand)?.Response);
+            Assert.Equal("hello world", (plan.Commands[0] as PredictedSayCommand)?.Response.Content);
+            Assert.Equal("test intent", (plan.Commands[0] as PredictedSayCommand)?.Response.Context?.Intent);
+            Assert.Equal("content", (plan.Commands[0] as PredictedSayCommand)?.Response.Context?.Citations[0].Content);
+            Assert.Equal("title", (plan.Commands[0] as PredictedSayCommand)?.Response.Context?.Citations[0].Title);
+            Assert.Equal("url", (plan.Commands[0] as PredictedSayCommand)?.Response.Context?.Citations[0].Url);
         }
 
         [Fact]

diff --git a/...rosoft.TeamsAI/Microsoft.TeamsAI.Tests/AITests/Augmentations/SequenceAugmentationTests.cs b/...rosoft.TeamsAI/Microsoft.TeamsAI.Tests/AITests/Augmentations/SequenceAugmentationTests.cs
@@ -49,7 +49,15 @@ public async Task Test_CreatePlanFromResponseAsync_ValidPlan_ShouldSucceed()
       ""response"": ""hello""
     }
   ]
-}"
+}",
+                    Context = new()
+                    {
+                        Intent = "test intent",
+                        Citations = new List<Citation>
+                        {
+                            new("content", "title", "url")
+                        }
+                    }
                 }
             };
 
@@ -62,7 +70,11 @@ public async Task Test_CreatePlanFromResponseAsync_ValidPlan_ShouldSucceed()
             Assert.Equal("DO", plan.Commands[0].Type);
             Assert.Equal("test", (plan.Commands[0] as PredictedDoCommand)?.Action);
             Assert.Equal("SAY", plan.Commands[1].Type);
-            Assert.Equal("hello", (plan.Commands[1] as PredictedSayCommand)?.Response);
+            Assert.Equal("hello", (plan.Commands[1] as PredictedSayCommand)?.Response.Content);
+            Assert.Equal("test intent", (plan.Commands[1] as PredictedSayCommand)?.Response.Context?.Intent);
+            Assert.Equal("content", (plan.Commands[1] as PredictedSayCommand)?.Response.Context?.Citations[0].Content);
+            Assert.Equal("title", (plan.Commands[1] as PredictedSayCommand)?.Response.Context?.Citations[0].Title);
+            Assert.Equal("url", (plan.Commands[1] as PredictedSayCommand)?.Response.Context?.Citations[0].Url);
         }
 
         [Fact]

diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Action/DefaultActions.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Action/DefaultActions.cs
@@ -81,11 +81,11 @@ public async Task<string> SayCommandAsync([ActionTurnContext] ITurnContext turnC
 
             if (turnContext.Activity.ChannelId == Channels.Msteams)
             {
-                await turnContext.SendActivityAsync(command.Response.Replace("\n", "<br>"), null, null, cancellationToken);
+                await turnContext.SendActivityAsync(command.Response.Content.Replace("\n", "<br>"), null, null, cancellationToken);
             }
             else
             {
-                await turnContext.SendActivityAsync(command.Response, null, null, cancellationToken);
+                await turnContext.SendActivityAsync(command.Response.Content, null, null, cancellationToken);
             };
 
             return string.Empty;

diff --git a/...et/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Augmentations/MonologueAugmentation.cs b/...et/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Augmentations/MonologueAugmentation.cs
@@ -260,7 +260,13 @@ public MonologueAugmentation(List<ChatCompletionAction> actions)
                         }
                     }
 
-                    command = new PredictedSayCommand(text);
+                    ChatMessage message = response.Message ?? new ChatMessage(ChatRole.Assistant)
+                    {
+                        Context = response.Message?.Context,
+                    };
+
+                    message.Content = text;
+                    command = new PredictedSayCommand(message);
                 }
                 else
                 {

diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Augmentations/SequenceAugmentation.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Augmentations/SequenceAugmentation.cs
@@ -48,6 +48,24 @@ public SequenceAugmentation(List<ChatCompletionAction> actions)
             try
             {
                 Plan? plan = JsonSerializer.Deserialize<Plan>(response.Message?.Content ?? "");
+
+                if (plan != null)
+                {
+                    foreach (IPredictedCommand cmd in plan.Commands)
+                    {
+                        if (cmd is PredictedSayCommand say)
+                        {
+                            ChatMessage message = response.Message ?? new ChatMessage(ChatRole.Assistant)
+                            {
+                                Context = response.Message?.Context,
+                            };
+
+                            message.Content = say.Response.Content;
+                            say.Response = message;
+                        }
+                    }
+                }
+
                 return await Task.FromResult(plan);
             }
             catch (Exception)

diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/MessageContext.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Models/MessageContext.cs
@@ -10,7 +10,7 @@ public class MessageContext
         /// <summary>
         /// Citations used in the message.
         /// </summary>
-        public IList<Citation> Citations { get; } = new List<Citation>();
+        public IList<Citation> Citations { get; set; } = new List<Citation>();
 
         /// <summary>
         /// The intent of the message.
@@ -31,7 +31,7 @@ public class Citation
         /// <summary>
         /// The title of the citation.
         /// </summary>
-        public string Title {  get; set; }
+        public string Title { get; set; }
 
         /// <summary>
         /// The URL of the citation.

diff --git a/.../packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Moderator/AzureContentSafetyModerator.cs b/.../packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Moderator/AzureContentSafetyModerator.cs
@@ -56,7 +56,7 @@ public async Task<Plan> ReviewOutputAsync(ITurnContext turnContext, TState turnS
                     {
                         if (command is PredictedSayCommand sayCommand)
                         {
-                            string output = sayCommand.Response;
+                            string output = sayCommand.Response.Content;
 
                             // If plan is flagged it will be replaced
                             Plan? newPlan = await _HandleTextModeration(output, false);

diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Moderator/OpenAIModerator.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Moderator/OpenAIModerator.cs
@@ -65,7 +65,7 @@ public async Task<Plan> ReviewOutputAsync(ITurnContext turnContext, TState turnS
                     {
                         if (command is PredictedSayCommand sayCommand)
                         {
-                            string output = sayCommand.Response;
+                            string output = sayCommand.Response.Content;
 
                             // If plan is flagged it will be replaced
                             Plan? newPlan = await _HandleTextModerationAsync(output, false, cancellationToken);

diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Planners/PredictedSayCommand.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Planners/PredictedSayCommand.cs
@@ -1,5 +1,7 @@
 using Json.Schema;
 using System.Text.Json.Serialization;
+using Microsoft.Teams.AI.AI.Models;
+using Microsoft.Teams.AI.Utilities.JsonConverters;
 
 namespace Microsoft.Teams.AI.AI.Planners
 {
@@ -17,19 +19,28 @@ public class PredictedSayCommand : IPredictedCommand
         /// The response that the AI system should say.
         /// </summary>
         [JsonPropertyName("response")]
+        [JsonConverter(typeof(ChatMessageJsonConverter))]
         [JsonRequired]
-        public string Response { get; set; }
+        public ChatMessage Response { get; set; }
 
         /// <summary>
         /// Creates a new instance of the <see cref="PredictedSayCommand"/> class.
         /// </summary>
         /// <param name="response">The response that the AI system should say.</param>
         [JsonConstructor]
-        public PredictedSayCommand(string response)
+        public PredictedSayCommand(ChatMessage response)
         {
             Response = response;
         }
 
+        public PredictedSayCommand(string response)
+        {
+            Response = new ChatMessage(ChatRole.Assistant)
+            {
+                Content = response
+            };
+        }
+
         /// <summary>
         /// Schema
         /// </summary>

diff --git a/.../Microsoft.TeamsAI/Microsoft.TeamsAI/Utilities/JsonConverters/ChatMessageJsonConverter.cs b/.../Microsoft.TeamsAI/Microsoft.TeamsAI/Utilities/JsonConverters/ChatMessageJsonConverter.cs
@@ -0,0 +1,33 @@
+using Microsoft.Teams.AI.AI.Models;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.Teams.AI.Utilities.JsonConverters
+{
+    internal class ChatMessageJsonConverter : JsonConverter<ChatMessage>
+    {
+        public override ChatMessage Read(
+            ref Utf8JsonReader reader,
+            Type typeToConvert,
+            JsonSerializerOptions options)
+        {
+            string? response = JsonSerializer.Deserialize<string>(ref reader);
+
+            if (response == null)
+            {
+                throw new JsonException();
+            }
+
+            return new ChatMessage(ChatRole.Assistant)
+            {
+                Content = response
+            };
+        }
+
+        public override void Write(Utf8JsonWriter writer, ChatMessage value, JsonSerializerOptions options)
+        {
+            writer.WriteStringValue(value.Content);
+            writer.Flush();
+        }
+    }
+}
diff --git a/...ages/Microsoft.TeamsAI/Microsoft.TeamsAI/Utilities/JsonConverters/CommandJsonConverter.cs b/...ages/Microsoft.TeamsAI/Microsoft.TeamsAI/Utilities/JsonConverters/CommandJsonConverter.cs
@@ -83,7 +83,7 @@ public override void Write(Utf8JsonWriter writer, IPredictedCommand value, JsonS
 
                 writer.WritePropertyName(_responsePropertyName);
 
-                JsonSerializer.Serialize(writer, ((PredictedSayCommand)value).Response, options);
+                JsonSerializer.Serialize(writer, ((PredictedSayCommand)value).Response.Content, options);
             }
             else
             {
-Original file line number
+Diff line change
@@ Expand Up @@
                     writer.WritePropertyName(_responsePropertyName);
-                    JsonSerializer.Serialize(writer, ((PredictedSayCommand)value).Response, options);
+                    JsonSerializer.Serialize(writer, ((PredictedSayCommand)value).Response.Content, options);
                 }
                 else
                 {
@@ Expand Down @@