Integrating Stepwise planner (#121)

### Motivation and Context

<!-- Thank you for your contribution to the copilot-chat repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

This PR adds support for Stepwise Planner to enable MRKL style planning
in Chat Copilot.
See microsoft/semantic-kernel#1468 for details
on how Stepwise Planner works.
- Stepwise Planner will only be called if a plugin has been enabled.
This follows the same logic as Action/Sequential Planner.
- Added `StepwiseStepView` and `StepwiseThoughtProcess` components to
render friendly view of Stepwise thought process in prompt dialog.

### Description

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

If Planner returns with thought process:

![image](https://github.com/microsoft/chat-copilot/assets/125500434/ef8b2dee-4bfb-44f8-a363-b8469196ebce)


![image](https://github.com/microsoft/chat-copilot/assets/125500434/88cdc5a9-ba9f-4341-bd13-80d3f1532951)

![image](https://github.com/microsoft/chat-copilot/assets/125500434/72027814-7085-4346-944d-999284e84a12)

If Planner returns with suggested response or information 

![image](https://github.com/microsoft/chat-copilot/assets/125500434/31a2b011-7590-4d82-86a3-a1e5799e019b)

Specific Changes:
- CopilotChatPlanner class:
- The planner now includes support for the new Stepwise Planner and
StepwiseStep model.
- The planner now includes bailout functionality, which allows the
planner to request additional user input if needed.
- ProposedPlan model:
  - Updated to include a new PlanType called Stepwise.
- PlannerOptions:
  - Updated to include a new StepwisePlannerConfig property.
- ChatMemoryController:
  - Updated to sanitize log input by removing new line characters.
- ExternalInformationSkill class:
- Updated to use the Stepwise Planner if the PlannerOptions.Type is set
to Stepwise.
- appsettings.json file:
- Updated to include a new configuration section, StepwisePlannerConfig,
which includes settings for the Stepwise Planner feature.
- TextUtils file:
- Added utility function to format text containing `\n` line breaks into
paragraphs.
  
### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [Contribution
Guidelines](https://github.com/microsoft/copilot-chat/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/copilot-chat/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
~~- [ ] All unit tests pass, and I have added new tests where possible~~
- [x] I didn't break anyone 😄

webapi/CopilotChatWebApi.csproj

@@ -13,6 +13,7 @@
     <PackageReference Include="Azure.AI.FormRecognizer" Version="4.0.0" />
     <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.35.2" />
     <PackageReference Include="Microsoft.SemanticKernel" Version="0.19.230804.2-preview" />
+    <PackageReference Include="Microsoft.SemanticKernel.Planning.StepwisePlanner" Version="0.19.230804.2-preview" />
     <PackageReference Include="Microsoft.SemanticKernel.Connectors.AI.OpenAI" Version="0.19.230804.2-preview" />
     <PackageReference Include="Microsoft.SemanticKernel.Connectors.Memory.AzureCognitiveSearch" Version="0.19.230804.2-preview" />
     <PackageReference Include="Microsoft.SemanticKernel.Connectors.Memory.Chroma" Version="0.19.230804.2-preview" />

webapi/Models/Response/ProposedPlan.cs

@@ -10,6 +10,7 @@ public enum PlanType
 {
     Action, // single-step
     Sequential, // multi-step
+    Stepwise, // MRKL style planning
 }
 
 // State of Plan

webapi/Options/PlannerOptions.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft. All rights reserved.
 using System.ComponentModel.DataAnnotations;
 using CopilotChat.WebApi.Models.Response;
+using Microsoft.SemanticKernel.Planning.Stepwise;
 
 namespace CopilotChat.WebApi.Options;
 
@@ -52,4 +53,10 @@ public class MissingFunctionErrorOptions
     /// Whether to retry plan creation if LLM returned response that doesn't contain valid plan (e.g., invalid XML or JSON, contains missing function, etc.).
     /// </summary>
     public bool AllowRetriesOnInvalidPlan { get; set; } = true;
+
+    /// <summary>
+    /// The configuration for the stepwise planner.
+    /// </summary>
+    [RequiredOnPropertyValue(nameof(Type), PlanType.Stepwise)]
+    public StepwisePlannerConfig StepwisePlannerConfig { get; set; } = new StepwisePlannerConfig();
 }

webapi/Skills/ChatSkills/CopilotChatPlanner.cs

@@ -2,13 +2,15 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using CopilotChat.WebApi.Models.Response;
 using CopilotChat.WebApi.Options;
 using Microsoft.Extensions.Logging;
 using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Orchestration;
 using Microsoft.SemanticKernel.Planning;
 using Microsoft.SemanticKernel.Planning.Sequential;
 using Microsoft.SemanticKernel.SkillDefinition;
@@ -39,15 +41,21 @@ public class CopilotChatPlanner
     /// Flag to indicate that a variable is unknown and needs to be filled in by the user.
     /// This is used to flag any inputs that had dependencies from removed steps.
     /// </summary>
-    private const string UNKNOWN_VARIABLE_FLAG = "$???";
+    private const string UnknownVariableFlag = "$???";
 
     /// <summary>
     /// Regex to match variable names from plan parameters.
     /// Valid variable names can contain letters, numbers, underscores, and dashes but can't start with a number.
     /// Matches: $variableName, $variable_name, $variable-name, $some_variable_Name, $variableName123, $variableName_123, $variableName-123
     /// Does not match: $123variableName, $100 $200
     /// </summary>
-    private const string VARIABLE_REGEX = @"\$([A-Za-z]+[_-]*[\w]+)";
+    private const string VariableRegex = @"\$([A-Za-z]+[_-]*[\w]+)";
+
+    /// <summary>
+    /// Supplemental text to add to the plan goal if PlannerOptions.Type is set to Stepwise.
+    /// Helps the planner know when to bail out to request additional user input.
+    /// </summary>
+    private const string StepwisePlannerSupplement = "If you need more information to fulfill this request, return with a request for additional user input.";
 
     /// <summary>
     /// Initializes a new instance of the <see cref="CopilotChatPlanner"/> class.
@@ -74,21 +82,65 @@ public async Task<Plan> CreatePlanAsync(string goal, ILogger logger)
             return new Plan(goal);
         }
 
-        Plan plan = this._plannerOptions?.Type == PlanType.Sequential
-               ? await new SequentialPlanner(
+        Plan plan;
+
+        switch (this._plannerOptions?.Type)
+        {
+            case PlanType.Sequential:
+                plan = await new SequentialPlanner(
                     this.Kernel,
                     new SequentialPlannerConfig
                     {
                         RelevancyThreshold = this._plannerOptions?.RelevancyThreshold,
                         // Allow plan to be created with missing functions
                         AllowMissingFunctions = this._plannerOptions?.MissingFunctionError.AllowRetries ?? false
                     }
-                ).CreatePlanAsync(goal)
-               : await new ActionPlanner(this.Kernel).CreatePlanAsync(goal);
+                ).CreatePlanAsync(goal);
+                break;
+            default:
+                plan = await new ActionPlanner(this.Kernel).CreatePlanAsync(goal);
+                break;
+        }
 
         return this._plannerOptions!.MissingFunctionError.AllowRetries ? this.SanitizePlan(plan, plannerFunctionsView, logger) : plan;
     }
 
+    /// <summary>
+    /// Run the stepwise planner.
+    /// </summary>
+    /// <param name="goal">The goal containing user intent and ask context.</param>
+    /// <param name="context">The context to run the plan in.</param>
+    public async Task<SKContext> RunStepwisePlannerAsync(string goal, SKContext context)
+    {
+        var config = new Microsoft.SemanticKernel.Planning.Stepwise.StepwisePlannerConfig()
+        {
+            MaxTokens = this._plannerOptions?.StepwisePlannerConfig.MaxTokens ?? 2048,
+            MaxIterations = this._plannerOptions?.StepwisePlannerConfig.MaxIterations ?? 15,
+            MinIterationTimeMs = this._plannerOptions?.StepwisePlannerConfig.MinIterationTimeMs ?? 1500
+        };
+
+        Stopwatch sw = new();
+        sw.Start();
+
+        try
+        {
+            var plan = new StepwisePlanner(
+                this.Kernel,
+                config
+            ).CreatePlan(string.Join("\n", goal, StepwisePlannerSupplement));
+            var result = await plan.InvokeAsync(context);
+
+            sw.Stop();
+            result.Variables.Set("timeTaken", sw.Elapsed.ToString());
+            return result;
+        }
+        catch (Exception e)
+        {
+            context.Log.LogError(e, "Error running stepwise planner");
+            throw;
+        }
+    }
+
     #region Private
 
     /// <summary>
@@ -112,7 +164,7 @@ private Plan SanitizePlan(Plan plan, FunctionsView availableFunctions, ILogger l
                 availableOutputs.AddRange(step.Outputs);
 
                 // Regex to match variable names
-                Regex variableRegEx = new(VARIABLE_REGEX, RegexOptions.Singleline);
+                Regex variableRegEx = new(VariableRegex, RegexOptions.Singleline);
 
                 // Check for any inputs that may have dependencies from removed steps
                 foreach (var input in step.Parameters)
@@ -133,7 +185,7 @@ private Plan SanitizePlan(Plan plan, FunctionsView availableFunctions, ILogger l
                                     && inputVariableMatch.Groups[1].Captures.Count == 1
                                     && !unavailableOutputs.Any(output => string.Equals(output, inputVariableValue, StringComparison.OrdinalIgnoreCase))
                                         ? "$PLAN.RESULT" // TODO: [Issue #2256] Extract constants from Plan class, requires change on kernel team
-                                        : UNKNOWN_VARIABLE_FLAG;
+                                        : UnknownVariableFlag;
                                 step.Parameters.Set(input.Key, Regex.Replace(input.Value, variableRegEx.ToString(), overrideValue));
                             }
                         }

webapi/Skills/ChatSkills/ExternalInformationSkill.cs

@@ -85,6 +85,15 @@ public async Task<string> AcquireExternalInformationAsync(
             return string.Empty;
         }
 
+        var contextString = string.Join("\n", context.Variables.Where(v => v.Key != "userIntent").Select(v => $"{v.Key}: {v.Value}"));
+        var goal = $"Given the following context, accomplish the user intent.\nContext:\n{contextString}\nUser Intent:{userIntent}";
+        if (this._planner.PlannerOptions?.Type == PlanType.Stepwise)
+        {
+            var newPlanContext = context.Clone();
+            newPlanContext = await this._planner.RunStepwisePlannerAsync(goal, context);
+            return $"{PromptPreamble}\n{newPlanContext.Variables.Input.Trim()}\n{PromptPostamble}\n";
+        }
+
         // Check if plan exists in ask's context variables.
         var planExists = context.Variables.TryGetValue("proposedPlan", out string? proposedPlanJson);
         var deserializedPlan = planExists && !string.IsNullOrWhiteSpace(proposedPlanJson) ? JsonSerializer.Deserialize<ProposedPlan>(proposedPlanJson) : null;
@@ -127,7 +136,6 @@ public async Task<string> AcquireExternalInformationAsync(
         else
         {
             // Create a plan and set it in context for approval.
-            var contextString = string.Join("\n", context.Variables.Where(v => v.Key != "userIntent").Select(v => $"{v.Key}: {v.Value}"));
             Plan? plan = null;
             // Use default planner options if planner options are null.
             var plannerOptions = this._planner.PlannerOptions ?? new PlannerOptions();
@@ -139,7 +147,7 @@ public async Task<string> AcquireExternalInformationAsync(
             { // TODO: [Issue #2256] Remove InvalidPlan retry logic once Core team stabilizes planner
                 try
                 {
-                    plan = await this._planner.CreatePlanAsync($"Given the following context, accomplish the user intent.\nContext:\n{contextString}\nUser Intent:{userIntent}", context.Logger);
+                    plan = await this._planner.CreatePlanAsync(goal, context.Logger);
                 }
                 catch (Exception e) when (this.IsRetriableError(e))
                 {

webapi/appsettings.json

@@ -49,15 +49,17 @@
   //
   // Planner can determine which skill functions, if any, need to be used to fulfill a user's request.
   // https://learn.microsoft.com/en-us/semantic-kernel/concepts-sk/planner
-  // - Set Planner:Type to "Action" to use the single-step ActionPlanner (default)
+  // - Set Planner:Type to "Action" to use the single-step ActionPlanner
   // - Set Planner:Type to "Sequential" to enable the multi-step SequentialPlanner
   //     Note: SequentialPlanner works best with `gpt-4`. See the "Enabling Sequential Planner" section in webapi/README.md for configuration instructions.
   // - Set Planner:RelevancyThreshold to a decimal between 0 and 1.0. 
   //
   "Planner": {
     "Type": "Sequential",
-    // Set RelevancyThreshold to a value >= 0.50 if using the SequentialPlanner with gpt-3.5-turbo. Ignored when Planner:Type is "Action"
-    "RelevancyThreshold": "0.80",
+    // The minimum relevancy score for a function to be considered.
+    // Set RelevancyThreshold to a value between 0 and 1 if using the SequentialPlanner or Stepwise planner with gpt-3.5-turbo. 
+    // Ignored when Planner:Type is "Action"
+    "RelevancyThreshold": "0.25",
     // Whether to allow missing functions in the plan on creation then sanitize output. Functions are considered missing if they're not available in the planner's kernel's context.
     // If set to true, the plan will be created with missing functions as no-op steps that are filtered from the final proposed plan.
     // If this is set to false, the plan creation will fail if any functions are missing.
@@ -66,7 +68,12 @@
       "MaxRetriesAllowed": "3" // Max retries allowed on MissingFunctionsError. If set to 0, no retries will be attempted.
     },
     // Whether to retry plan creation if LLM returned response with invalid plan.
-    "AllowRetriesOnInvalidPlan": "true"
+    "AllowRetriesOnInvalidPlan": "true",
+    "StepwisePlannerConfig": {
+      "MaxTokens": "2048",
+      "MaxIterations": "15",
+      "MinIterationTimeMs": "1500"
+    }
   },
   //
   // Optional Azure Speech service configuration for providing Azure Speech access tokens.

webapp/src/Constants.ts

@@ -53,4 +53,5 @@ export const Constants = {
         MANIFEST_PATH: '/.well-known/ai-plugin.json',
     },
     KEYSTROKE_DEBOUNCE_TIME_MS: 250,
+    STEPWISE_RESULT_NOT_FOUND_REGEX: /(Result not found, review _stepsTaken to see what happened\.)\s+(\[{.*}])/g,
 };

webapp/src/components/chat/prompt-dialog/PromptDialog.tsx

@@ -19,10 +19,13 @@ import {
 } from '@fluentui/react-components';
 import { Info16Regular } from '@fluentui/react-icons';
 import React from 'react';
+import { Constants } from '../../../Constants';
 import { BotResponsePrompt, PromptSectionsNameMap } from '../../../libs/models/BotResponsePrompt';
 import { IChatMessage } from '../../../libs/models/ChatMessage';
 import { useDialogClasses } from '../../../styles';
 import { TokenUsageGraph } from '../../token-usage/TokenUsageGraph';
+import { formatParagraphTextContent } from '../../utils/TextUtils';
+import { StepwiseThoughtProcess } from './stepwise-planner/StepwiseThoughtProcess';
 
 const useClasses = makeStyles({
     prompt: {
@@ -50,18 +53,20 @@ export const PromptDialog: React.FC<IPromptDialogProps> = ({ message }) => {
     } catch (e) {
         prompt = message.prompt ?? '';
     }
-
     let promptDetails;
     if (typeof prompt === 'string') {
         promptDetails = prompt.split('\n').map((paragraph, idx) => <p key={`prompt-details-${idx}`}>{paragraph}</p>);
     } else {
         promptDetails = Object.entries(prompt).map(([key, value]) => {
+            const isStepwiseThoughtProcess = Constants.STEPWISE_RESULT_NOT_FOUND_REGEX.test(value as string);
             return value ? (
                 <div className={classes.prompt} key={`prompt-details-${key}`}>
                     <Body1Strong>{PromptSectionsNameMap[key]}</Body1Strong>
-                    {(value as string).split('\n').map((paragraph, idx) => (
-                        <p key={`prompt-details-${idx}`}>{paragraph}</p>
-                    ))}
+                    {isStepwiseThoughtProcess ? (
+                        <StepwiseThoughtProcess stepwiseResult={value as string} />
+                    ) : (
+                        formatParagraphTextContent(value as string)
+                    )}
                 </div>
             ) : null;
         });

webapp/src/components/chat/prompt-dialog/stepwise-planner/StepwiseStepView.tsx

@@ -0,0 +1,94 @@
+import {
+    AccordionHeader,
+    AccordionItem,
+    AccordionPanel,
+    Body1,
+    makeStyles,
+    shorthands,
+    tokens,
+} from '@fluentui/react-components';
+import { StepwiseStep } from '../../../../libs/models/StepwiseStep';
+import { formatParagraphTextContent } from '../../../utils/TextUtils';
+
+const useClasses = makeStyles({
+    root: {
+        display: 'flex',
+        ...shorthands.gap(tokens.spacingHorizontalM),
+    },
+    accordionItem: {
+        width: '99%',
+    },
+    header: {
+        width: '100%',
+        /* Styles for the button within the header */
+        '& button': {
+            alignItems: 'flex-start',
+            minHeight: '-webkit-fill-available',
+            paddingLeft: tokens.spacingHorizontalNone,
+        },
+    },
+});
+
+interface IStepwiseStepViewProps {
+    step: StepwiseStep;
+    index: number;
+}
+
+export const StepwiseStepView: React.FC<IStepwiseStepViewProps> = ({ step, index }) => {
+    const classes = useClasses();
+
+    let header = `[OBSERVATION] ${step.observation}`;
+    let details: string | undefined;
+
+    if (step.thought) {
+        const thoughtRegEx = /\[(THOUGHT|QUESTION|ACTION)](\s*(.*))*/g;
+        let thought = step.thought.match(thoughtRegEx)?.[0] ?? `[THOUGHT] ${step.thought}`;
+
+        // Only show the first sentence of the thought in the header.
+        // Show the rest as details.
+        const firstSentenceIndex = thought.indexOf('. ');
+        if (firstSentenceIndex > 0) {
+            details = thought.substring(firstSentenceIndex + 2);
+            thought = thought.substring(0, firstSentenceIndex + 1);
+        }
+
+        header = thought;
+    }
+
+    if (step.action) {
+        header = `[ACTION] ${step.action}`;
+
+        // Format the action variables and observation.
+        const variables = step.action_variables
+            ? 'Action variables: \n' +
+              Object.entries(step.action_variables)
+                  .map(([key, value]) => `\r${key}: ${value}`)
+                  .join('\n')
+            : '';
+
+        // Remove the [ACTION] tag from the thought and remove any code block formatting.
+        details = step.thought.replace('[ACTION]', '').replaceAll('```', '') + '\n';
+
+        // Parse any unicode quotation characters in the observation.
+        const observation = step.observation?.replaceAll(/\\{0,2}u0022/g, '"');
+        details = details.concat(variables, `\nObservation: \n\r${observation}`);
+    }
+
+    return (
+        <div className={classes.root}>
+            <Body1>{index + 1}.</Body1>
+            <AccordionItem value={index} className={classes.accordionItem}>
+                {details ? (
+                    <>
+                        <AccordionHeader expandIconPosition="end" className={classes.header}>
+                            <Body1>{header}</Body1>
+                        </AccordionHeader>
+                        <AccordionPanel>{formatParagraphTextContent(details)}</AccordionPanel>
+                    </>
+                ) : (
+                    <Body1>{header}</Body1>
+                )}
+            </AccordionItem>
+        </div>
+    );
+};