diff --git a/src/main/resources/cloudapim/extensions/ai/AiProvidersPage.js b/src/main/resources/cloudapim/extensions/ai/AiProvidersPage.js index e6fe165..9d1a23a 100644 --- a/src/main/resources/cloudapim/extensions/ai/AiProvidersPage.js +++ b/src/main/resources/cloudapim/extensions/ai/AiProvidersPage.js @@ -232,15 +232,39 @@ class AiProvidersPage extends Component { type: 'number', props: { label: 'Context size' }, }, - 'deny': { + 'regex_validation.deny': { type: 'array', props: { label: 'Deny', suffix: 'regex' }, }, - 'allow': { + 'regex_validation.allow': { type: 'array', props: { label: 'Allow', suffix: 'regex' }, }, - 'validator_ref': { + 'http_validation.url': { + type: 'string', + props: { label: 'URL' }, + }, + 'http_validation.headers': { + type: 'object', + props: { label: 'Headers' }, + }, + 'http_validation.ttl': { + type: 'object', + props: { label: 'TTL', suffix: 'millis.' }, + }, + 'cache.ttl': { + type: 'number', + props: { label: 'TTL', suffix: 'millis.' }, + }, + 'cache.strategy': { + type: 'select', + props: { label: 'Cache strategy', possibleValues: [ + { label: 'None', value: 'none' }, + { label: 'Simple', value: 'simple' }, + { label: 'Semantic', value: 'semantic' }, + ] }, + }, + 'llm_validation.provider': { type: 'select', props: { label: 'Validator provider', @@ -252,7 +276,7 @@ class AiProvidersPage extends Component { }), } }, - 'validator_prompt': { + 'llm_validation.prompt': { type: 'select', props: { label: 'Validator prompt', @@ -284,11 +308,19 @@ class AiProvidersPage extends Component { 'description', '<<>>Cache', + 'cache.strategy', + 'cache.ttl', + '>>>Regex validation', + 'regex_validation.allow', + 'regex_validation.deny', + '>>>LLM Based validation', + 'llm_validation.provider', + 'llm_validation.prompt', + '>>>External validation', + 'http_validation.url', + 'http_validation.headers', + 'http_validation.ttl', '>>>Metadata and tags', 'tags', 'metadata', @@ -316,11 +348,19 @@ class AiProvidersPage extends Component { 'options.num_gpu', 'options.num_gqa', 'options.num_ctx', - '<<>>Cache', + 'cache.strategy', + 'cache.ttl', + '>>>Regex validation', + 'regex_validation.allow', + 'regex_validation.deny', + '>>>LLM Based validation', + 'llm_validation.provider', + 'llm_validation.prompt', + '>>>External validation', + 'http_validation.url', + 'http_validation.headers', + 'http_validation.ttl', '>>>Tester', 'tester', '>>>Metadata and tags', @@ -344,11 +384,19 @@ class AiProvidersPage extends Component { 'options.safe_prompt', 'options.temperature', 'options.top_p', - '<<>>Cache', + 'cache.strategy', + 'cache.ttl', + '>>>Regex validation', + 'regex_validation.allow', + 'regex_validation.deny', + '>>>LLM Based validation', + 'llm_validation.provider', + 'llm_validation.prompt', + '>>>External validation', + 'http_validation.url', + 'http_validation.headers', + 'http_validation.ttl', '>>>Tester', 'tester', '>>>Metadata and tags', @@ -371,11 +419,19 @@ class AiProvidersPage extends Component { 'options.temperature', 'options.top_p', 'options.top_k', - '<<>>Cache', + 'cache.strategy', + 'cache.ttl', + '>>>Regex validation', + 'regex_validation.allow', + 'regex_validation.deny', + '>>>LLM Based validation', + 'llm_validation.provider', + 'llm_validation.prompt', + '>>>External validation', + 'http_validation.url', + 'http_validation.headers', + 'http_validation.ttl', '>>>Tester', 'tester', '>>>Metadata and tags', @@ -398,11 +454,19 @@ class AiProvidersPage extends Component { 'options.n', 'options.temperature', 'options.topP', - '<<>>Cache', + 'cache.strategy', + 'cache.ttl', + '>>>Regex validation', + 'regex_validation.allow', + 'regex_validation.deny', + '>>>LLM Based validation', + 'llm_validation.provider', + 'llm_validation.prompt', + '>>>External validation', + 'http_validation.url', + 'http_validation.headers', + 'http_validation.ttl', '>>>Tester', 'tester', '>>>Metadata and tags', @@ -424,11 +488,19 @@ class AiProvidersPage extends Component { 'options.n', 'options.temperature', 'options.topP', - '<<>>Cache', + 'cache.strategy', + 'cache.ttl', + '>>>Regex validation', + 'regex_validation.allow', + 'regex_validation.deny', + '>>>LLM Based validation', + 'llm_validation.provider', + 'llm_validation.prompt', + '>>>External validation', + 'http_validation.url', + 'http_validation.headers', + 'http_validation.ttl', '>>>Tester', 'tester', '>>>Metadata and tags', diff --git a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/decorators.scala b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/decorators.scala index 44fd475..78479aa 100644 --- a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/decorators.scala +++ b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/decorators.scala @@ -8,6 +8,10 @@ object ChatClientDecorators { val possibleDecorators: Seq[Function[(AiProvider, ChatClient), ChatClient]] = Seq( ChatClientWithRegexValidation.applyIfPossible, ChatClientWithLlmValidation.applyIfPossible, + ChatClientWithSimpleCache.applyIfPossible, + ChatClientWithSemanticCache.applyIfPossible, + ChatClientWithSemanticCache.applyIfPossible, + ChatClientWithHttpValidation.applyIfPossible, ) def apply(provider: AiProvider, client: ChatClient): ChatClient = { diff --git a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/externalvalidation.scala b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/externalvalidation.scala index 455d6d9..a29eea9 100644 --- a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/externalvalidation.scala +++ b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/externalvalidation.scala @@ -1,5 +1,64 @@ package com.cloud.apim.otoroshi.extensions.aigateway.decorators -class externalvalidation { +import com.cloud.apim.otoroshi.extensions.aigateway.{ChatClient, ChatPrompt, ChatResponse} +import com.cloud.apim.otoroshi.extensions.aigateway.entities.AiProvider +import com.github.blemale.scaffeine.Scaffeine +import otoroshi.env.Env +import otoroshi.utils.TypedMap +import play.api.libs.json.{JsValue, Json} +import otoroshi.utils.syntax.implicits._ +import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.duration.FiniteDuration + +object ChatClientWithHttpValidation { + val cache = Scaffeine() + .expireAfter[String, (FiniteDuration, Boolean)]( + create = (key, value) => value._1, + update = (key, value, currentDuration) => currentDuration, + read = (key, value, currentDuration) => currentDuration + ) + .maximumSize(5000) + .build[String, (FiniteDuration, Boolean)]() + def applyIfPossible(tuple: (AiProvider, ChatClient)): ChatClient = { + if (tuple._1.httpValidation.url.isDefined) { + new ChatClientWithHttpValidation(tuple._1, tuple._2) + } else { + tuple._2 + } + } } + +class ChatClientWithHttpValidation(originalProvider: AiProvider, chatClient: ChatClient) extends ChatClient { + + private val ttl = originalProvider.httpValidation.ttl + + + override def call(originalPrompt: ChatPrompt, attrs: TypedMap)(implicit ec: ExecutionContext, env: Env): Future[Either[JsValue, ChatResponse]] = { + + def pass(): Future[Either[JsValue, ChatResponse]] = chatClient.call(originalPrompt, attrs) + + def fail(): Future[Either[JsValue, ChatResponse]] = Left(Json.obj("error" -> "bad_request", "error_description" -> s"request content did not pass http validation")).vfuture + + val key = originalPrompt.messages.map(m => s"${m.role}:${m.content}").mkString(",").sha512 + ChatClientWithHttpValidation.cache.getIfPresent(key) match { + case Some((_, true)) => pass() + case Some((_, false)) => fail() + case None => { + env.Ws + .url(originalProvider.httpValidation.url.get) + .withHttpHeaders(originalProvider.httpValidation.headers.toSeq: _*) + .post(originalPrompt.json).flatMap { resp => + if (resp.status != 200) { + ChatClientWithHttpValidation.cache.put(key, (ttl, false)) + fail() + } else { + val value = resp.json.select("result").asOpt[Boolean].getOrElse(false) + ChatClientWithHttpValidation.cache.put(key, (ttl, value)) + pass() + } + } + } + } + } +} \ No newline at end of file diff --git a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/llmvalidation.scala b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/llmvalidation.scala index e99d455..1fd69ba 100644 --- a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/llmvalidation.scala +++ b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/llmvalidation.scala @@ -12,7 +12,7 @@ import scala.concurrent.{ExecutionContext, Future} object ChatClientWithLlmValidation { def applyIfPossible(tuple: (AiProvider, ChatClient)): ChatClient = { - if (tuple._1.validatorRef.isDefined && tuple._1.validatorPrompt.isDefined) { + if (tuple._1.llmValidation.provider.isDefined && tuple._1.llmValidation.prompt.isDefined) { new ChatClientWithLlmValidation(tuple._1, tuple._2) } else { tuple._2 @@ -28,11 +28,11 @@ class ChatClientWithLlmValidation(originalProvider: AiProvider, chatClient: Chat def fail(idx: Int): Future[Either[JsValue, ChatResponse]] = Left(Json.obj("error" -> "bad_request", "error_description" -> s"request content did not pass llm validation (${idx})")).vfuture - originalProvider.validatorRef match { + originalProvider.llmValidation.provider match { case None => pass() case Some(ref) if ref == originalProvider.id => pass() case Some(ref) => { - originalProvider.validatorPrompt match { + originalProvider.llmValidation.prompt match { case None => pass() case Some(pref) => env.adminExtensions.extension[AiExtension].flatMap(_.states.prompt(pref)) match { case None => Left(Json.obj("error" -> "validation prompt not found")).vfuture diff --git a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/regexvalidation.scala b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/regexvalidation.scala index cf4f921..b06462e 100644 --- a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/regexvalidation.scala +++ b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/regexvalidation.scala @@ -11,7 +11,7 @@ import scala.concurrent.{ExecutionContext, Future} object ChatClientWithRegexValidation { def applyIfPossible(tuple: (AiProvider, ChatClient)): ChatClient = { - if (tuple._1.allow.nonEmpty || tuple._1.deny.nonEmpty) { + if (tuple._1.regexValidation.allow.nonEmpty || tuple._1.regexValidation.deny.nonEmpty) { new ChatClientWithRegexValidation(tuple._1, tuple._2) } else { tuple._2 @@ -21,8 +21,8 @@ object ChatClientWithRegexValidation { class ChatClientWithRegexValidation(originalProvider: AiProvider, chatClient: ChatClient) extends ChatClient { - private val allow = originalProvider.allow - private val deny = originalProvider.deny + private val allow = originalProvider.regexValidation.allow + private val deny = originalProvider.regexValidation.deny private def validate(content: String): Boolean = { val allowed = if (allow.isEmpty) true else allow.exists(al => RegexPool.regex(al).matches(content)) diff --git a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/semanticcache.scala b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/semanticcache.scala index b8116b1..396e939 100644 --- a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/semanticcache.scala +++ b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/semanticcache.scala @@ -2,6 +2,7 @@ package com.cloud.apim.otoroshi.extensions.aigateway.decorators import com.cloud.apim.otoroshi.extensions.aigateway.{ChatClient, ChatGeneration, ChatMessage, ChatPrompt, ChatResponse, ChatResponseMetadata} import com.cloud.apim.otoroshi.extensions.aigateway.entities.AiProvider +import com.github.benmanes.caffeine.cache.RemovalCause import com.github.blemale.scaffeine.Scaffeine import dev.langchain4j.data.segment.TextSegment import dev.langchain4j.model.embedding.onnx.allminilml6v2.AllMiniLmL6V2EmbeddingModel @@ -17,7 +18,6 @@ import scala.concurrent.{ExecutionContext, Future} import scala.concurrent.duration.{DurationInt, FiniteDuration} import scala.jdk.CollectionConverters._ - object ChatClientWithSemanticCache { val embeddingStores = new TrieMap[String, InMemoryEmbeddingStore[TextSegment]]() val embeddingModel = new AllMiniLmL6V2EmbeddingModel() @@ -27,10 +27,22 @@ object ChatClientWithSemanticCache { update = (key, value, currentDuration) => currentDuration, read = (key, value, currentDuration) => currentDuration ) - .maximumSize(5000) + .maximumSize(5000) // TODO: custom ? .build[String, (FiniteDuration, ChatResponse)]() + + val cacheEmbeddingCleanup = Scaffeine() + .expireAfter[String, (FiniteDuration, Function[String, Unit])]( + create = (key, value) => value._1, + update = (key, value, currentDuration) => currentDuration, + read = (key, value, currentDuration) => currentDuration + ) + .removalListener((key: String, value: (FiniteDuration, Function[String, Unit]), reason: RemovalCause) => { + value._2(key) + }) + .maximumSize(5000) // TODO: custom ? + .build[String, (FiniteDuration, Function[String, Unit])]() def applyIfPossible(tuple: (AiProvider, ChatClient)): ChatClient = { - if (tuple._1.cacheStrategy.contains("semantic")) { + if (tuple._1.cache.strategy.contains("semantic")) { new ChatClientWithSemanticCache(tuple._1, tuple._2) } else { tuple._2 @@ -40,14 +52,14 @@ object ChatClientWithSemanticCache { class ChatClientWithSemanticCache(originalProvider: AiProvider, chatClient: ChatClient) extends ChatClient { - private val ttl = originalProvider.ttl.getOrElse(24.hours) + private val ttl = originalProvider.cache.ttl override def call(originalPrompt: ChatPrompt, attrs: TypedMap)(implicit ec: ExecutionContext, env: Env): Future[Either[JsValue, ChatResponse]] = { val query = originalPrompt.messages.filter(_.role.toLowerCase().trim == "user").map(_.content).mkString(", ") val key = query.sha512 ChatClientWithSemanticCache.cache.getIfPresent(key) match { case Some((_, response)) => - println("using semantic cached response") + // println("using semantic cached response") response.rightf case None => { val embeddingModel = ChatClientWithSemanticCache.embeddingModel @@ -55,13 +67,13 @@ class ChatClientWithSemanticCache(originalProvider: AiProvider, chatClient: Chat new InMemoryEmbeddingStore[TextSegment]() } val queryEmbedding = embeddingModel.embed(query).content() - val relevant = embeddingStore.search(EmbeddingSearchRequest.builder().queryEmbedding(queryEmbedding).maxResults(1).build()) + val relevant = embeddingStore.search(EmbeddingSearchRequest.builder().queryEmbedding(queryEmbedding).maxResults(1).minScore(0.7).build()) val matches = relevant.matches().asScala if (matches.nonEmpty) { val resp = matches.head - val id = resp.embeddingId() val text = resp.embedded().text() - println("using semantic response") + // val score = resp.score() + // println(s"using semantic response: ${score}") val chatResponse = ChatResponse(Seq(ChatGeneration(ChatMessage("assistant", text))), ChatResponseMetadata.empty) ChatClientWithSemanticCache.cache.put(key, (ttl, chatResponse)) chatResponse.rightf @@ -73,6 +85,9 @@ class ChatClientWithSemanticCache(originalProvider: AiProvider, chatClient: Chat val embedding = embeddingModel.embed(segment).content() embeddingStore.add(key, embedding, segment) ChatClientWithSemanticCache.cache.put(key, (ttl, resp)) + ChatClientWithSemanticCache.cacheEmbeddingCleanup.put(key, (ttl, (key) => { + embeddingStore.remove(key) + })) resp.right } } diff --git a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/simplecache.scala b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/simplecache.scala index 2487a78..311ec3b 100644 --- a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/simplecache.scala +++ b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/decorators/simplecache.scala @@ -21,7 +21,7 @@ object ChatClientWithSimpleCache { .maximumSize(5000) .build[String, (FiniteDuration, ChatResponse)]() def applyIfPossible(tuple: (AiProvider, ChatClient)): ChatClient = { - if (tuple._1.cacheStrategy.contains("simple")) { + if (tuple._1.cache.strategy.contains("simple")) { new ChatClientWithSimpleCache(tuple._1, tuple._2) } else { tuple._2 @@ -31,13 +31,13 @@ object ChatClientWithSimpleCache { class ChatClientWithSimpleCache(originalProvider: AiProvider, chatClient: ChatClient) extends ChatClient { - private val ttl = originalProvider.ttl.getOrElse(24.hours) + private val ttl = originalProvider.cache.ttl override def call(originalPrompt: ChatPrompt, attrs: TypedMap)(implicit ec: ExecutionContext, env: Env): Future[Either[JsValue, ChatResponse]] = { val key = originalPrompt.messages.map(m => s"${m.role}:${m.content}").mkString(",").sha512 ChatClientWithSimpleCache.cache.getIfPresent(key) match { case Some((_, response)) => - println("using simple cache response") + // println("using simple cache response") response.rightf case None => { chatClient.call(originalPrompt, attrs).map { diff --git a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/entities/provider.scala b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/entities/provider.scala index 5b9eecc..63ea32a 100644 --- a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/entities/provider.scala +++ b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/entities/provider.scala @@ -1,7 +1,7 @@ package com.cloud.apim.otoroshi.extensions.aigateway.entities +import com.cloud.apim.otoroshi.extensions.aigateway.ChatClient import com.cloud.apim.otoroshi.extensions.aigateway.decorators.ChatClientDecorators -import com.cloud.apim.otoroshi.extensions.aigateway.{ChatClient, ChatClientWithValidation} import com.cloud.apim.otoroshi.extensions.aigateway.providers._ import otoroshi.api.{GenericResourceAccessApiWithState, Resource, ResourceVersion} import otoroshi.env.Env @@ -17,6 +17,27 @@ import java.util.concurrent.TimeUnit import scala.concurrent.duration._ import scala.util.{Failure, Success, Try} +case class RegexValidationSettings( + allow: Seq[String] = Seq.empty, + deny: Seq[String] = Seq.empty, +) + +case class LlmValidationSettings( + provider: Option[String] = None, + prompt: Option[String] = None, +) + +case class HttpValidationSettings( + url: Option[String] = None, + headers: Map[String, String] = Map.empty, + ttl: FiniteDuration = 5.minutes, +) + + +case class CacheSettings( + strategy: String = "none", + ttl: FiniteDuration = 24.hours +) case class AiProvider( location: EntityLocation, @@ -28,12 +49,10 @@ case class AiProvider( provider: String, connection: JsObject, options: JsObject, - allow: Seq[String] = Seq.empty, - deny: Seq[String] = Seq.empty, - validatorRef: Option[String] = None, - validatorPrompt: Option[String] = None, - cacheStrategy: Option[String],// = None, - ttl: Option[FiniteDuration],// = None, + regexValidation: RegexValidationSettings = RegexValidationSettings(), + llmValidation: LlmValidationSettings = LlmValidationSettings(), + httpValidation: HttpValidationSettings = HttpValidationSettings(), + cache: CacheSettings = CacheSettings(), ) extends EntityLocationSupport { override def internalId: String = id override def json: JsValue = AiProvider.format.writes(this) @@ -91,10 +110,23 @@ object AiProvider { "provider" -> o.provider, "connection" -> o.connection, "options" -> o.options, - "allow" -> o.allow, - "deny" -> o.deny, - "validator_ref" -> o.validatorRef, - "validator_prompt" -> o.validatorPrompt, + "regex_validation" -> Json.obj( + "allow" -> o.regexValidation.allow, + "deny" -> o.regexValidation.deny, + ), + "llm_validation" -> Json.obj( + "provider" -> o.llmValidation.provider, + "prompt" -> o.llmValidation.prompt, + ), + "http_validation" -> Json.obj( + "url" -> o.httpValidation.url, + "headers" -> o.httpValidation.headers, + "ttl" -> o.httpValidation.ttl.toMillis, + ), + "cache" -> Json.obj( + "strategy" -> o.cache.strategy, + "ttl" -> o.cache.ttl.toMillis, + ) ) override def reads(json: JsValue): JsResult[AiProvider] = Try { AiProvider( @@ -107,10 +139,23 @@ object AiProvider { provider = (json \ "provider").as[String], connection = (json \ "connection").asOpt[JsObject].getOrElse(Json.obj()), options = (json \ "options").asOpt[JsObject].getOrElse(Json.obj()), - allow = (json \ "allow").asOpt[Seq[String]].getOrElse(Seq.empty), - deny = (json \ "deny").asOpt[Seq[String]].getOrElse(Seq.empty), - validatorRef = (json \ "validator_ref").asOpt[String], - validatorPrompt = (json \ "validator_prompt").asOpt[String], + regexValidation = RegexValidationSettings( + allow = (json \ "regex_validation" \ "allow").asOpt[Seq[String]].getOrElse(Seq.empty), + deny = (json \ "regex_validation" \ "deny").asOpt[Seq[String]].getOrElse(Seq.empty), + ), + llmValidation = LlmValidationSettings( + provider = (json \ "llm_validation" \ "provider").asOpt[String], + prompt = (json \ "llm_validation" \ "prompt").asOpt[String], + ), + httpValidation = HttpValidationSettings( + url = (json \ "http_validation" \ "url").asOpt[String], + headers = (json \ "http_validation" \ "headers").asOpt[Map[String, String]].getOrElse(Map.empty), + ttl = (json \ "http_validation" \ "ttl").asOpt[Long].map(v => FiniteDuration(v, TimeUnit.MILLISECONDS)).getOrElse(5.minutes), + ), + cache = CacheSettings( + strategy = (json \ "cache" \ "strategy").asOpt[String].getOrElse("none"), + ttl = (json \ "cache" \ "ttl").asOpt[Long].map(v => FiniteDuration(v, TimeUnit.MILLISECONDS)).getOrElse(24.hours), + ) ) } match { case Failure(ex) => JsError(ex.getMessage) diff --git a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/models.scala b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/models.scala index f8db5e7..b3255e2 100644 --- a/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/models.scala +++ b/src/main/scala/com/cloud/apim/otoroshi/extensions/aigateway/models.scala @@ -45,7 +45,6 @@ case class ChatResponse( } case class ChatResponseMetadata(rateLimit: ChatResponseMetadataRateLimit, usage: ChatResponseMetadataUsage) { - val empty: ChatResponseMetadata = ChatResponseMetadata(ChatResponseMetadataRateLimit.empty, ChatResponseMetadataUsage.empty) def json: JsValue = Json.obj( "rate_limit" -> rateLimit.json, "usage" -> usage.json,