[C#] feat: Powered by AI: Citations, feedback loop, and GeneratedByAI…

… icon (#1648) ## Linked issues closes: #1539 ![image](https://github.com/microsoft/teams-ai/assets/115390646/528f68c9-fd52-4df5-aba1-415a983a1adb) ## Details - Add citations support - Add Powered by AI (generated by AI) support - Add enableFeedbackLoop support - Add documentations on powered by AI UX features in getting started folder - NO unit tests: Will open a fast follow PR. ## Attestation Checklist - [x] My code follows the style guidelines of this project - I have checked for/fixed spelling, linting, and other errors - I have commented my code for clarity - I have made corresponding changes to the documentation (updating the doc strings in the code is sufficient) - My changes generate no new warnings - I have added tests that validates my changes, and provides sufficient test coverage. I have tested with: - Local testing - E2E testing in Teams - New and existing unit tests pass locally with my changes --------- Co-authored-by: Alex Acebo <[email protected]>
microsoft · May 15, 2024 · bac1461 · bac1461
1 parent 58fdeab
commit bac1461
Show file tree

Hide file tree

Showing 15 changed files with 576 additions and 21 deletions.
diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/AI.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/AI.cs
@@ -40,7 +40,7 @@ public AI(AIOptions<TState> options, ILoggerFactory? loggerFactory = null)
             _actions = new ActionCollection<TState>();
 
             // Import default actions
-            ImportActions(new DefaultActions<TState>(loggerFactory));
+            ImportActions(new DefaultActions<TState>(options.EnableFeedbackLoop, loggerFactory));
         }
 
         /// <summary>

diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/AIOptions.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/AIOptions.cs
@@ -56,6 +56,12 @@ public sealed class AIOptions<TState> where TState : TurnState
         /// </remarks>
         public bool? AllowLooping { get; set; }
 
+        /// <summary>
+        /// Optional. If true, the AI system will enable the feedback loop in Teams that allows a user to give thumbs up or down to a response.
+        /// Defaults to "false".
+        /// </summary>
+        public bool EnableFeedbackLoop { get; set; } = false;
+
         /// <summary>
         /// Initializes a new instance of the <see cref="AIOptions{TState}"/> class.
         /// </summary>

diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Action/AIEntity.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Action/AIEntity.cs
@@ -0,0 +1,204 @@
+using Microsoft.Bot.Schema;
+using Newtonsoft.Json;
+
+namespace Microsoft.Teams.AI.AI.Action
+{
+    /// <summary>
+    /// The citations's AIEntity.
+    /// </summary>
+    public class AIEntity : Entity
+    {
+        /// <summary>
+        /// Required. Must be "https://schema.org/Message"
+        /// </summary>
+        [JsonProperty(PropertyName = "type")]
+        public new string Type = "https://schema.org/Message";
+
+        /// <summary>
+        /// Required. Must be "Message".
+        /// </summary>
+        [JsonProperty(PropertyName = "@type")]
+        public string AtType = "Message";
+
+        /// <summary>
+        /// Required. Must be "https://schema.org"
+        /// </summary>
+        [JsonProperty(PropertyName = "@context")]
+        public string AtContext = "https://schema.org";
+
+        /// <summary>
+        /// Must be left blank. This is for Bot Framework's schema.
+        /// </summary>
+        [JsonProperty(PropertyName = "@id")]
+        public string AtId = "";
+
+        /// <summary>
+        /// Indicate that the content was generated by AI.
+        /// </summary>
+        [JsonProperty(PropertyName = "additionalType")]
+        public List<string> AdditionalType = new() { "AIGeneratedContent" };
+
+        /// <summary>
+        /// Optional. If the citation object is included, then the sent activity will include citations that are referenced in the activity text.
+        /// </summary>
+        [JsonProperty(PropertyName = "citation")]
+        public List<ClientCitation> Citation { get; set; } = new();
+    }
+
+    /// <summary>
+    /// The client citation.
+    /// </summary>
+    public class ClientCitation
+    {
+        /// <summary>
+        /// Required. Must be "Claim".
+        /// </summary>
+        [JsonProperty(PropertyName = "@type")]
+        public string AtType = "Claim";
+
+        /// <summary>
+        /// Required. Number and position of the citation.
+        /// </summary>
+        [JsonProperty(PropertyName = "position")]
+        public string Position { get; set; } = string.Empty;
+
+        /// <summary>
+        /// The citation's appearance.
+        /// </summary>
+        [JsonProperty(PropertyName = "appearance")]
+        public ClientCitationAppearance? Appearance { get; set; }
+
+    }
+
+    /// <summary>
+    /// The client citation appearance.
+    /// </summary>
+    public class ClientCitationAppearance
+    {
+        /// <summary>
+        /// Required. Must be "DigitalDocument"
+        /// </summary>
+        [JsonProperty(PropertyName = "@type")]
+        public string AtType = "DigitalDocument";
+
+        /// <summary>
+        /// Name of the document.
+        /// </summary>
+        [JsonProperty(PropertyName = "name")]
+        public string Name { get; set; } = string.Empty;
+
+        /// <summary>
+        /// Optional. The citation appreance text. It is ignored in Teams.
+        /// </summary>
+        [JsonProperty(PropertyName = "text")]
+        public string? Text { get; set; }
+
+        /// <summary>
+        /// URL of the document. This will make the name of the citation clickable and direct the user to the specified URL.
+        /// </summary>
+        [JsonProperty(PropertyName = "url")]
+        public string? Url { get; set; }
+
+        /// <summary>
+        /// Content of the citation. Should be clipped if longer than ~500 characters.
+        /// </summary>
+        [JsonProperty(PropertyName = "abstract")]
+        public string Abstract { get; set; } = string.Empty;
+
+        /// <summary>
+        /// The encoding format used for the icon.
+        /// </summary>
+        [JsonProperty(PropertyName = "encodingFormat")]
+        public string EncodingFormat { get; set; } = "text/html";
+
+        /// <summary>
+        /// The icon provided in the citation ui.
+        /// </summary>
+        [JsonProperty(PropertyName = "image")]
+        public string? Image { get; set; }
+
+        /// <summary>
+        /// Optional. Set the keywords.
+        /// </summary>
+        [JsonProperty(PropertyName = "keywords")]
+        public List<string>? Keywords { get; set; }
+
+        /// <summary>
+        /// Optional sensitivity content information.
+        /// </summary>
+        [JsonProperty(PropertyName = "usageInfo")]
+        public SensitivityUsageInfo? UsageInfo { get; set; }
+    }
+
+    /// <summary>
+    /// The sensitivity usage info.
+    /// </summary>
+    public class SensitivityUsageInfo
+    {
+        /// <summary>
+        /// Must be "https://schema.org/Message"
+        /// </summary>
+        [JsonProperty(PropertyName = "type")]
+        public string Type = "https://schema.org/Message";
+
+        /// <summary>
+        /// Required. Set to "CreativeWork".
+        /// </summary>
+        [JsonProperty(PropertyName = "@type")]
+        public string AtType = "CreativeWork";
+
+        /// <summary>
+        /// Sensitivity description of the content.
+        /// </summary>
+        [JsonProperty(PropertyName = "description")]
+        public string? Description { get; set; }
+
+        /// <summary>
+        /// Sensitivity title of the content.
+        /// </summary>
+        [JsonProperty(PropertyName = "name")]
+        public string? Name { get; set; }
+
+        /// <summary>
+        /// Optional. Ignored in Teams
+        /// </summary>
+        [JsonProperty(PropertyName = "position")]
+        public int Position { get; set; }
+
+        /// <summary>
+        /// The sensitivity usage info pattern.
+        /// </summary>
+        [JsonProperty(PropertyName = "pattern")]
+        public SensitivityUsageInfoPattern? Pattern;
+    }
+
+    /// <summary>
+    /// The sensitivity usage info pattern.
+    /// </summary>
+    public class SensitivityUsageInfoPattern
+    {
+        /// <summary>
+        /// Set to "DefinedTerm".
+        /// </summary>
+        [JsonProperty(PropertyName = "@type")]
+        public string AtType = "DefinedTerm";
+
+        /// <summary>
+        /// Whether it's in a defined term set.
+        /// </summary>
+        [JsonProperty(PropertyName = "inDefinedTermSet")]
+        public string? inDefinedTermSet { get; set; }
+
+        /// <summary>
+        /// The color.
+        /// </summary>
+        [JsonProperty(PropertyName = "name")]
+        public string? Name { get; set; }
+
+        /// <summary>
+        /// For example `#454545`.
+        /// </summary>
+        [JsonProperty(PropertyName = "termCode")]
+        public string? TermCode { get; set; }
+    }
+}
diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Action/DefaultActions.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Action/DefaultActions.cs
@@ -6,15 +6,19 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Bot.Builder;
 using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Teams.AI.AI.Models;
+using Microsoft.Bot.Schema;
 
 namespace Microsoft.Teams.AI.AI.Action
 {
     internal class DefaultActions<TState> where TState : TurnState
     {
         private readonly ILogger _logger;
+        private readonly bool _enableFeedbackLoop;
 
-        public DefaultActions(ILoggerFactory? loggerFactory = null)
+        public DefaultActions(bool enableFeedbackLoop = false, ILoggerFactory? loggerFactory = null)
         {
+            _enableFeedbackLoop = enableFeedbackLoop;
             _logger = loggerFactory is null ? NullLogger.Instance : loggerFactory.CreateLogger(typeof(DefaultActions<TState>));
         }
 
@@ -79,14 +83,71 @@ public async Task<string> SayCommandAsync([ActionTurnContext] ITurnContext turnC
             Verify.ParamNotNull(command);
             Verify.ParamNotNull(command.Response);
 
-            if (turnContext.Activity.ChannelId == Channels.Msteams)
+            if (command.Response.Content == null || command.Response.Content == string.Empty)
             {
-                await turnContext.SendActivityAsync(command.Response.Content.Replace("\n", "<br>"), null, null, cancellationToken);
+                return "";
             }
-            else
+
+            string content = command.Response.Content;
+
+            bool isTeamsChannel = turnContext.Activity.ChannelId == Channels.Msteams;
+
+            if (isTeamsChannel)
+            {
+                content.Replace("\n", "<br>");
+            }
+
+            // If the response from the AI includes citations, those citations will be parsed and added to the SAY command.
+            List<ClientCitation> citations = new();
+
+            if (command.Response.Context != null && command.Response.Context.Citations.Count > 0)
+            {
+                int i = 0;
+                foreach (Citation citation in command.Response.Context.Citations)
+                {
+                    string abs = CitationUtils.Snippet(citation.Content, 500);
+                    if (isTeamsChannel)
+                    {
+                        content.Replace("\n", "<br>");
+                    };
+
+                    citations.Add(new ClientCitation()
+                    {
+                        Position = $"{i + 1}",
+                        Appearance = new ClientCitationAppearance()
+                        {
+                            Name = citation.Title,
+                            Abstract = abs
+                        }
+                    });
+                    i++;
+                }
+            }
+
+            // If there are citations, modify the content so that the sources are numbers instead of [doc1], [doc2], etc.
+            string contentText = citations.Count == 0 ? content : CitationUtils.FormatCitationsResponse(content);
+
+            // If there are citations, filter out the citations unused in content.
+            List<ClientCitation>? referencedCitations = citations.Count > 0 ? CitationUtils.GetUsedCitations(contentText, citations) : new List<ClientCitation>();
+
+            object? channelData = isTeamsChannel ? new
+            {
+                feedbackLoopEnabled = _enableFeedbackLoop
+            } : null;
+
+            AIEntity entity = new();
+            if (referencedCitations != null)
+            {
+                entity.Citation = referencedCitations;
+            }
+
+            await turnContext.SendActivityAsync(new Activity()
             {
-                await turnContext.SendActivityAsync(command.Response.Content, null, null, cancellationToken);
-            };
+                Type = ActivityTypes.Message,
+                Text = contentText,
+                ChannelData = channelData,
+                Entities = new List<Entity>() { entity }
+            }, cancellationToken);
 
             return string.Empty;
         }

diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Augmentations/DefaultAugmentation.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Augmentations/DefaultAugmentation.cs
@@ -1,4 +1,5 @@
 using Microsoft.Bot.Builder;
+using Microsoft.Teams.AI.AI.Models;
 using Microsoft.Teams.AI.AI.Planners;
 using Microsoft.Teams.AI.AI.Prompts;
 using Microsoft.Teams.AI.AI.Prompts.Sections;
@@ -30,12 +31,22 @@ public DefaultAugmentation()
         /// <inheritdoc />
         public async Task<Plan?> CreatePlanFromResponseAsync(ITurnContext context, IMemory memory, PromptResponse response, CancellationToken cancellationToken = default)
         {
-            return await Task.FromResult(new Plan()
+            PredictedSayCommand say = new(response.Message?.Content ?? "");
+
+            if (response.Message != null)
             {
-                Commands =
+                ChatMessage message = new(ChatRole.Assistant)
                 {
-                    new PredictedSayCommand(response.Message?.Content ?? "")
-                }
+                    Context = response.Message!.Context,
+                    Content = response.Message.Content
+                };
+
+                say.Response = message;
+            }
+
+            return await Task.FromResult(new Plan()
+            {
+                Commands = { say }
             });
         }