OpenAIModerator.ts

/**
 * @module teams-ai
 */
/**
 * Copyright (c) Microsoft Corporation. All rights reserved.
 * Licensed under the MIT License.
 */

import { Plan, PredictedDoCommand, PredictedSayCommand } from '../planners';
import { TurnState } from '../TurnState';
import { TurnContext } from 'botbuilder';
import {
    OpenAIClient,
    CreateModerationResponseResultsInner,
    CreateModerationResponse,
    OpenAIClientResponse
} from '../internals';
import { Moderator } from './Moderator';
import { AI } from '../AI';

/**
 * Options for the OpenAI based moderator.
 */
export interface OpenAIModeratorOptions {
    /**
     * OpenAI API key
     */
    apiKey: string;

    /**
     * Which parts of the conversation to moderate.
     */
    moderate: 'input' | 'output' | 'both';

    /**
     * Optional. OpenAI organization.
     */
    organization?: string;

    /**
     * Optional. OpenAI endpoint.
     */
    endpoint?: string;

    /**
     * Optional. OpenAI model to use.
     */
    model?: string;

    /**
     * Optional. Azure Content Safety API version.
     */
    apiVersion?: string;
}

/**
 * A moderator that uses OpenAI's moderation API to review prompts and plans for safety.
 * @remarks
 * This moderation can be configure to review the input from the user, output from the model, or both.
 * @template TState Optional. Type of the applications turn state.
 */
export class OpenAIModerator<TState extends TurnState = TurnState> implements Moderator<TState> {
    private readonly _options: OpenAIModeratorOptions;
    private readonly _client: OpenAIClient;

    /**
     * Creates a new instance of the OpenAI based moderator.
     * @param {OpenAIModeratorOptions} options Configuration options for the moderator.
     */
    public constructor(options: OpenAIModeratorOptions) {
        this._options = Object.assign({}, options);
        this._client = this.createClient(this._options);
    }

    /**
     * Reviews an incoming utterance for safety violations.
     * @param {TurnContext} context - Context for the current turn of conversation.
     * @param {TState} state - Application state for the current turn of conversation.
     * @returns {Promise<Plan | undefined>} An undefined value to approve the prompt or a new plan to redirect to if not approved.
     */
    public async reviewInput(context: TurnContext, state: TState): Promise<Plan | undefined> {
        switch (this._options.moderate) {
            case 'input':
            case 'both': {
                const input = state.temp.input ?? context.activity.text;
                const result = await this.createModeration(input, this._options.model);
                if (result) {
                    if (result.flagged) {
                        // Input flagged
                        return {
                            type: 'plan',
                            commands: [
                                {
                                    type: 'DO',
                                    action: AI.FlaggedInputActionName,
                                    parameters: result
                                } as PredictedDoCommand
                            ]
                        };
                    }
                } else {
                    // Rate limited
                    return {
                        type: 'plan',
                        commands: [{ type: 'DO', action: AI.HttpErrorActionName, parameters: {} } as PredictedDoCommand]
                    };
                }
                break;
            }
        }
        return undefined;
    }

    /**
     * Reviews the SAY commands generated by the planner for safety violations.
     * @param {TurnContext} context - Context for the current turn of conversation.
     * @param {TState} state - Application state for the current turn of conversation.
     * @param {Plan} plan - Plan generated by the planner.
     * @returns {Promise<Plan>} The plan to execute. Either the current plan passed in for review or a new plan.
     */
    public async reviewOutput(context: TurnContext, state: TState, plan: Plan): Promise<Plan> {
        switch (this._options.moderate) {
            case 'output':
            case 'both':
                for (let i = 0; i < plan.commands.length; i++) {
                    const cmd = plan.commands[i];
                    if (cmd.type == 'SAY') {
                        const output = (cmd as PredictedSayCommand).response;
                        const result = await this.createModeration(output.content || '', this._options.model);
                        if (result) {
                            if (result.flagged) {
                                // Output flagged
                                return {
                                    type: 'plan',
                                    commands: [
                                        {
                                            type: 'DO',
                                            action: AI.FlaggedOutputActionName,
                                            parameters: result
                                        } as PredictedDoCommand
                                    ]
                                };
                            }
                        } else {
                            // Rate limited
                            return {
                                type: 'plan',
                                commands: [
                                    { type: 'DO', action: AI.HttpErrorActionName, parameters: {} } as PredictedDoCommand
                                ]
                            };
                        }
                    }
                }
                break;
        }

        return plan;
    }
    public get options(): OpenAIModeratorOptions {
        return this._options;
    }

    protected createClient(options: OpenAIModeratorOptions): OpenAIClient {
        return new OpenAIClient({
            apiKey: options.apiKey,
            organization: options.organization,
            endpoint: options.endpoint
        });
    }

    protected async createModeration(
        input: string,
        model?: string
    ): Promise<CreateModerationResponseResultsInner | undefined> {
        const response = (await this._client.createModeration({
            input,
            model
        })) as OpenAIClientResponse<CreateModerationResponse>;
        if (response.data?.results && Array.isArray(response.data.results) && response.data.results.length > 0) {
            return response.data.results[0];
        } else {
            return undefined;
        }
    }
}