diff --git a/src/main/resources/cloudapim/extensions/ai/AiProvidersPage.js b/src/main/resources/cloudapim/extensions/ai/AiProvidersPage.js index 9d1a23a..4be930e 100644 --- a/src/main/resources/cloudapim/extensions/ai/AiProvidersPage.js +++ b/src/main/resources/cloudapim/extensions/ai/AiProvidersPage.js @@ -269,6 +269,20 @@ class AiProvidersPage extends Component { props: { label: 'Validator provider', placeholder: 'Select a validator provider', + isClearable: true, + valuesFrom: '/bo/api/proxy/apis/ai-gateway.extensions.cloud-apim.com/v1/providers', + transformer: (a) => ({ + value: a.id, + label: a.name, + }), + } + }, + 'provider_fallback': { + type: 'select', + props: { + label: 'Provider fallback provider', + placeholder: 'Select a fallback', + isClearable: true, valuesFrom: '/bo/api/proxy/apis/ai-gateway.extensions.cloud-apim.com/v1/providers', transformer: (a) => ({ value: a.id, @@ -281,6 +295,7 @@ class AiProvidersPage extends Component { props: { label: 'Validator prompt', placeholder: 'Select a validator prompt', + isClearable: true, valuesFrom: '/bo/api/proxy/apis/ai-gateway.extensions.cloud-apim.com/v1/prompts', transformer: (a) => ({ value: a.id, @@ -308,6 +323,8 @@ class AiProvidersPage extends Component { 'description', '<<>>Provider fallback', + 'provider_fallback', '>>>Cache', 'cache.strategy', 'cache.ttl', @@ -348,6 +365,8 @@ class AiProvidersPage extends Component { 'options.num_gpu', 'options.num_gqa', 'options.num_ctx', + '>>>Provider fallback', + 'provider_fallback', '>>>Cache', 'cache.strategy', 'cache.ttl', @@ -384,6 +403,8 @@ class AiProvidersPage extends Component { 'options.safe_prompt', 'options.temperature', 'options.top_p', + '>>>Provider fallback', + 'provider_fallback', '>>>Cache', 'cache.strategy', 'cache.ttl', @@ -419,6 +440,8 @@ class AiProvidersPage extends Component { 'options.temperature', 'options.top_p', 'options.top_k', + '>>>Provider fallback', + 'provider_fallback', '>>>Cache', 'cache.strategy', 'cache.ttl', @@ -454,6 +477,8 @@ class AiProvidersPage extends Component { 'options.n', 'options.temperature', 'options.topP', + '>>>Provider fallback', + 'provider_fallback', '>>>Cache', 'cache.strategy', 'cache.ttl', @@ -488,6 +513,8 @@ class AiProvidersPage extends Component { 'options.n', 'options.temperature', 'options.topP', + '>>>Provider fallback', + 'provider_fallback', '>>>Cache', 'cache.strategy', 'cache.ttl', diff --git a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/decorators.scala b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/decorators.scala index 78479aa..6c5266c 100644 --- a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/decorators.scala +++ b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/decorators.scala @@ -12,6 +12,7 @@ object ChatClientDecorators { ChatClientWithSemanticCache.applyIfPossible, ChatClientWithSemanticCache.applyIfPossible, ChatClientWithHttpValidation.applyIfPossible, + ChatClientWithProviderFallback.applyIfPossible ) def apply(provider: AiProvider, client: ChatClient): ChatClient = { diff --git a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/fallback.scala b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/fallback.scala new file mode 100644 index 0000000..0a5c777 --- /dev/null +++ b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/fallback.scala @@ -0,0 +1,49 @@ +package com.cloud.apim.otoroshi.extensions.aigateway.decorators + +import com.cloud.apim.otoroshi.extensions.aigateway.entities.AiProvider +import com.cloud.apim.otoroshi.extensions.aigateway.{ChatClient, ChatPrompt, ChatResponse} +import otoroshi.env.Env +import otoroshi.utils.TypedMap +import otoroshi.utils.syntax.implicits._ +import otoroshi_plugins.com.cloud.apim.extensions.aigateway.AiExtension +import play.api.libs.json.{JsValue, Json} + +import scala.concurrent.{ExecutionContext, Future} + +object ChatClientWithProviderFallback { + def applyIfPossible(tuple: (AiProvider, ChatClient)): ChatClient = { + if (tuple._1.providerFallback.isDefined) { + new ChatClientWithProviderFallback(tuple._1, tuple._2) + } else { + tuple._2 + } + } +} + +class ChatClientWithProviderFallback(originalProvider: AiProvider, chatClient: ChatClient) extends ChatClient { + + override def call(originalPrompt: ChatPrompt, attrs: TypedMap)(implicit ec: ExecutionContext, env: Env): Future[Either[JsValue, ChatResponse]] = { + chatClient.call(originalPrompt, attrs).flatMap { + case Left(err) => { + env.adminExtensions.extension[AiExtension].flatMap(_.states.provider(originalProvider.providerFallback.get)) match { + case None => err.leftf + case Some(provider) => provider.getChatClient() match { + case None => err.leftf + case Some(fallbackClient) => fallbackClient.call(originalPrompt, attrs) + } + } + } + case Right(resp) => resp.rightf + }.recoverWith { + case t: Throwable => { + env.adminExtensions.extension[AiExtension].flatMap(_.states.provider(originalProvider.providerFallback.get)) match { + case None => Json.obj("error" -> "fallback provider not found").leftf + case Some(provider) => provider.getChatClient() match { + case None => Json.obj("error" -> "fallback client not found").leftf + case Some(fallbackClient) => fallbackClient.call(originalPrompt, attrs) + } + } + } + } + } +} \ No newline at end of file diff --git a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/entities/provider.scala b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/entities/provider.scala index 63ea32a..5af9979 100644 --- a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/entities/provider.scala +++ b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/entities/provider.scala @@ -49,6 +49,7 @@ case class AiProvider( provider: String, connection: JsObject, options: JsObject, + providerFallback: Option[String] = None, regexValidation: RegexValidationSettings = RegexValidationSettings(), llmValidation: LlmValidationSettings = LlmValidationSettings(), httpValidation: HttpValidationSettings = HttpValidationSettings(), @@ -110,6 +111,7 @@ object AiProvider { "provider" -> o.provider, "connection" -> o.connection, "options" -> o.options, + "provider_fallback" -> o.providerFallback.map(_.json).getOrElse(JsNull).asValue, "regex_validation" -> Json.obj( "allow" -> o.regexValidation.allow, "deny" -> o.regexValidation.deny, @@ -139,6 +141,7 @@ object AiProvider { provider = (json \ "provider").as[String], connection = (json \ "connection").asOpt[JsObject].getOrElse(Json.obj()), options = (json \ "options").asOpt[JsObject].getOrElse(Json.obj()), + providerFallback = (json \ "provider_fallback").asOpt[String], regexValidation = RegexValidationSettings( allow = (json \ "regex_validation" \ "allow").asOpt[Seq[String]].getOrElse(Seq.empty), deny = (json \ "regex_validation" \ "deny").asOpt[Seq[String]].getOrElse(Seq.empty),