feat: store server api key on client side

Author: dumbbellcode

package-lock.json

@@ -1,5 +1,7 @@
 {
   "dependencies": {
+    "@google/generative-ai": "^0.24.1",
+    "@huggingface/transformers": "^3.6.3",
     "dom-accessibility-api": "^0.7.0",
     "marked": "^15.0.7",
     "notivue": "^2.4.5",

package.json

@@ -7,7 +7,14 @@ import {
   Settings,
   SelectInputContext,
 } from "@/types/common"
-import { getMessageForStatusCode } from "@/utils/auth"
+import {
+  buildMultiSelectInputDataExtractionPrompt,
+  buildSingleSelectInputDataExtractionPrompt,
+  buildTextInputDataExtractionPrompt,
+  wrapContext,
+} from "./services/prompts"
+import { AccurateExtractResult } from "./services/types"
+import Llmservice from "./services/llm"
 
 const API_URL = import.meta.env.VITE_API_URL
 const FILL_ACCURATE = `${API_URL}/fill/accurate`
@@ -18,12 +25,7 @@ export async function getAccurateFillData(
 ) {
   const settings = await getValueFromStorage<Settings>("settings", "sync")
   const activeProfileId = settings?.activeProfileId ?? "default"
-
-  const tokens = await getValueFromStorage<{ server: string }>(
-    "tokens",
-    "local",
-  )
-
+  const llmApiKey = settings.llmApiKey
   const userContext = await getValueFromStorage<AccurateDetails>(
     `${activeProfileId}-${DETAIL_TYPES.SHORT}`,
     "sync",
@@ -49,8 +51,12 @@ export async function getAccurateFillData(
     userContextCreativeFill,
   )
 
-  //console.info("User context", userTotalContext)
-  //console.info("Input context", textInputContext)
+  if (!llmApiKey) {
+    return {
+      success: false,
+      message: "Please add your Gemini API key from settings",
+    }
+  }
 
   if (!userTotalContext.length) {
     return {
@@ -59,45 +65,90 @@ export async function getAccurateFillData(
     }
   }
 
-  try {
-    const response = await fetch(FILL_ACCURATE, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        Authorization: `Bearer ${tokens?.server}`,
-      },
-      body: JSON.stringify({
-        context: userTotalContext,
-        text_input_context: textInputContext,
-        select_context: selectContext,
-      }),
-    })
 
-    const statusCode = response.status
-    if (!response.ok) {
-      if (statusCode === 401) {
-        chrome.storage.local.remove("tokens")
-      }
+  const llmService = new Llmservice(llmApiKey)
+  const extractedData = await passContextAndGetAnswers(
+    llmService,
+    userTotalContext,
+    textInputContext,
+    selectContext,
+  )
 
-      return {
-        success: false,
-        message: getMessageForStatusCode(statusCode),
-        error: "Network response was not ok",
-      }
-    }
+  return {
+    success: true,
+    data: extractedData,
+  }
+}
 
-    const data = await response.json()
+// LLMs cooking
+async function passContextAndGetAnswers(
+  llmService: Llmservice,
+  userContext: { label: string; value: string }[],
+  textInputContext: TextInputContext[],
+  selectContext: SelectInputContext[],
+) {
+  const contextPrompt = wrapContext(userContext)
+  let result: AccurateExtractResult[] = []
 
-    return {
-      success: true,
-      data,
-    }
-  } catch (error) {
-    console.info("fetch error", error)
-    return {
-      success: false,
-      message: getMessageForStatusCode(503),
-      error,
+  if (textInputContext.length) {
+    const textPrompt = buildTextInputDataExtractionPrompt(
+      contextPrompt,
+      textInputContext,
+    )
+    const schema: object = [
+      {
+        dataId: "abc-uuid",
+        value: "answer or null",
+      },
+    ]
+    const promptResult =
+      (await llmService.getResultWithSchema(textPrompt, schema)) ?? []
+    result = result.concat(promptResult as AccurateExtractResult[])
+  }
+
+  const singleSelectContext = []
+  const multiSelectContext = []
+  const singleSelectTags = ["option", "radio"]
+  for (const ctx of selectContext) {
+    if (singleSelectTags.includes(ctx.tagName ?? "")) {
+      singleSelectContext.push(ctx)
+    } else {
+      multiSelectContext.push(ctx)
     }
   }
+
+  if (singleSelectContext.length) {
+    const selectPrompt = buildSingleSelectInputDataExtractionPrompt(
+      contextPrompt,
+      singleSelectContext,
+    )
+    const schema: object = [
+      {
+        dataId: "abc-uuid",
+        value: "correct option",
+      },
+    ]
+    const promptResult =
+      (await llmService.getResultWithSchema(selectPrompt, schema)) ?? []
+    result = result.concat(promptResult as AccurateExtractResult[])
+  }
+
+  if (multiSelectContext.length) {
+    const selectPrompt = buildMultiSelectInputDataExtractionPrompt(
+      contextPrompt,
+      multiSelectContext,
+    )
+    const schema: object = [
+      {
+        dataId: "abc-uuid",
+        value: "correct option 1 | correct option 2",
+      },
+    ]
+
+    const promptResult =
+      (await llmService.getResultWithSchema(selectPrompt, schema)) ?? []
+    result = result.concat(promptResult as AccurateExtractResult[])
+  }
+
+  return result.filter((v) => v)
 }

src/background/service.ts

@@ -0,0 +1,73 @@
+import { GenerativeModel, GoogleGenerativeAI } from "@google/generative-ai"
+
+export default class Llmservice {
+  private genAI: GoogleGenerativeAI
+  private textModel: GenerativeModel
+  private jsonModel: GenerativeModel
+
+  constructor(apiKey: string) {
+    this.genAI = new GoogleGenerativeAI(apiKey)
+    // For text-only input, use the gemini-pro model
+    this.textModel = this.genAI.getGenerativeModel({
+      model: "gemini-2.5-flash",
+    })
+    // For JSON output, we'll use the same model and guide it with the prompt
+    this.jsonModel = this.genAI.getGenerativeModel({
+      model: "gemini-2.5-flash",
+    })
+  }
+
+  /**
+   * Generates a structured JSON object based on the message and schema.
+   * @param message The prompt message for the model.
+   * @param schema The desired JSON schema (provided as a string hint).
+   * @returns A promise that resolves to a JavaScript object.
+   */
+  async getResultWithSchema(
+    message: string,
+    schema: object,
+  ): Promise<object | null> {
+    const schemaHint = JSON.stringify(schema)
+    const fullPrompt = `${message}. Please provide the output in the following JSON format: ${schemaHint}. Do not include any additional text or explanations, only the JSON object.`
+
+    try {
+      const result = await this.jsonModel.generateContent(fullPrompt)
+      const response = await result.response
+      let text = response.text()
+      // Attempt to extract pure JSON by stripping markdown code block
+      text = text.trim()
+      if (text.startsWith("```json") && text.endsWith("```")) {
+        text = text.substring("```json".length).trim()
+        text = text.substring(0, text.length - "```".length).trim()
+      }
+      return JSON.parse(text)
+    } catch (error) {
+      console.error(
+        "Failed to generate or parse JSON from model output:",
+        error,
+      )
+      return null
+    }
+  }
+
+  /**
+   * Generates a plain text result from a given message.
+   * @param message The prompt message for the model.
+   * @returns A promise that resolves to the generated text string.
+   */
+  async getResult(message: string): Promise<string> {
+    try {
+      const result = await this.textModel.generateContent(message)
+      const response = await result.response
+
+      console.log({
+        question: message,
+        response: response.text(),
+      })
+      return response.text()
+    } catch (error) {
+      console.error("Failed to generate text from model:", error)
+      return ""
+    }
+  }
+}

src/background/services/llm.ts

@@ -0,0 +1,114 @@
+const OPTIONS_SEPARATOR = "|"
+
+function wrapFormDataPoint(obj: object, tag: string): string {
+  const start = `<${tag}>`
+  const end = `</${tag}>`
+  const dataString = Object.entries(obj)
+    .map(([key, val]) => {
+      if (Array.isArray(val)) {
+        return `${key}: ${val.join(OPTIONS_SEPARATOR)}`
+      }
+
+      return `${key}: ${val}`
+    })
+    .join("\n")
+  return [start, dataString, end].join("\n")
+}
+
+function wrapFormDataPoints(
+  obj: object[],
+  tag: string = "FormDataPoint",
+): string {
+  return obj.map((o) => wrapFormDataPoint(o, tag)).join("\n\n")
+}
+
+export function safeParseLLMJson(outputJson: string) {
+  try {
+    const json = JSON.parse(outputJson)
+    return json
+  } catch (err) {
+    console.error(err)
+    return null
+  }
+}
+
+export function wrapContext(
+  context: {
+    label: string
+    value: string
+  }[],
+) {
+  const start = "<Context>"
+  const end = "</Context>"
+  const contextString = context.reduce((prev, { label, value }) => {
+    return prev + `${label}:${value}\n`
+  }, "")
+  return [start, contextString, end].join("\n")
+}
+
+export function buildTextInputDataExtractionPrompt(
+  contextPrompt: string,
+  dataPoints: object[],
+) {
+  const dataPointText = wrapFormDataPoints(dataPoints)
+
+  return `
+You are an intelligent LLM that extracts data from context based on form input.
+Strictly respond with answer or null. Do not provide explanations.
+
+${contextPrompt}
+
+Provided below is extracted form input data, you have to pick the correct value from context 
+based on title, placeholder and label of form. If they are empty, try to find value based on
+closestLabel and closestText. Convert answer in any format if it's mentioned.
+If type is textarea, answer with statements. If type in input, answer accurately and precisely.
+
+
+${dataPointText}
+`
+}
+
+export function buildSingleSelectInputDataExtractionPrompt(
+  contextPrompt: string,
+  dataPoints: object[],
+): string {
+  const dataPointText = wrapFormDataPoints(dataPoints, "SingleChoice")
+
+  return `
+  You are an intelligent LLM that extracts data from context based on form input.
+  Strictly respond with correct option or null. Do not provide explanations.
+
+  ${contextPrompt}
+
+  Provided below is list of single choice questions <SingleChoice>. 
+  The question is present in title, placeholder, label, closestLabel or closestText. 
+  The options are separated by ${OPTIONS_SEPARATOR} character. Based on provided context and question, 
+  pick only one correct option, return null if the answer of question is not present 
+  in context or option. 
+
+  ${dataPointText}
+  `
+}
+
+export function buildMultiSelectInputDataExtractionPrompt(
+  contextPrompt: string,
+  dataPoints: object[],
+): string {
+  const dataPointText = wrapFormDataPoints(dataPoints, "MultipleChoice")
+
+  return `
+  You are an intelligent LLM that extracts data from context based on form input.
+  Strictly respond with correct option or null. Do not provide explanations.
+
+  ${contextPrompt}
+
+  Provided below is list of multiple choice questions <MultipleChoice>. 
+  The question is present in title, placeholder, label, closestLabel or closestText. 
+  The options are separated by ${OPTIONS_SEPARATOR} character.
+  Based on provided context and question, return an array of options
+  that are applicable for the questions, return null if no options apply for the 
+  context. 
+
+  ${dataPointText}
+  `
+}

src/background/services/prompts.ts

@@ -0,0 +1,4 @@
+export interface AccurateExtractResult {
+  dataId: string
+  value: string
+}