[QUO-758] Add Logs, Add Decorator, Add Hallucination Analysis (#65)

* Add Logs Resource, Add Decorator

* styling, fixes

* add boolean for hallucination

* readme format

* cleanup example api

* remove background task

* Remove decorator

* cleanup unused

* rename

* PR Feedback

* version

* cleanup

* name

* endpoints and api

* add rules

* Feedback

Author: mike-goitia

.gitignore

@@ -123,6 +123,7 @@ celerybeat.pid
 .env
 .env.local
 .venv
+.venv.*
 env/
 venv/
 ENV/

README.md

@@ -51,3 +51,32 @@ new_dataset = quotient.datasets.create(
 
 print(new_dataset)
 ```
+
+**Create a log with hallucination detection:**
+Log an event with hallucination detection. This will create a log event in Quotient and perform hallucination detection on the model output, input, and documents. This is a fire and forget operation, so it will not block the execution of your code.
+
+Additional examples can be found in the [examples](examples) directory.
+
+```python
+from quotientai import QuotientAI
+
+quotient = QuotientAI()
+quotient_logger = quotient.logger.init(
+    # Required
+    app_name="my-app",
+    environment="dev",
+    # dynamic labels for slicing/dicing analytics e.g. by customer, feature, etc
+    tags={"model": "gpt-4o", "feature": "customer-support"},
+    hallucination_detection=True,
+    inconsistency_detection=True,
+)
+
+quotient_logger.log(
+    model_input="Sample input",
+    model_output="Sample output",
+    # Documents from your retriever used to generate the model output
+    documents=[{"page_content": "Sample document"}], 
+    # optional additional context to help with hallucination detection, e.g. rules, constraints, etc
+    contexts=["Sample context"], 
+)
+```

examples/logging/example_fast_api/__init__.py

@@ -0,0 +1,23 @@
+########################################################
+# Fixed constants for demonstration
+########################################################
+PROMPT = """
+You are a helpful assistant that can answer questions about the context. Follow the rules provided if they are relevant.
+
+### Question
+{{question}}
+
+### Context
+{{context}}
+
+### Rules
+{{rules}}
+"""
+RETRIEVED_DOCUMENTS = [
+    {
+        "page_content": "Our company has unlimited vacation days",
+        "metadata": {"document_id": "123"},
+    }
+]
+QUESTION = "What is the company's vacation policy?"
+RULES = ["If you do not know the answer, just say that you do not know."]

examples/logging/example_fast_api/constants.py

@@ -0,0 +1,61 @@
+import os
+import chevron
+from fastapi import APIRouter
+from dotenv import load_dotenv
+from openai import OpenAI
+from quotientai import QuotientAI
+from constants import RETRIEVED_DOCUMENTS, QUESTION, PROMPT, RULES
+
+# Load environment variables
+load_dotenv()
+
+# Initialize OpenAI client
+client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
+
+########################################################
+# Initialize QuotientAI and QuotientAI Logger
+########################################################
+quotient = QuotientAI()
+quotient_logger = quotient.logger.init(
+    app_name="my-app",
+    environment="dev",
+    tags={"model": "gpt-4o", "feature": "customer-support"},
+    hallucination_detection=True,
+)
+
+# Create a router for the endpoint
+router = APIRouter()
+
+
+@router.post("/create-log/")
+async def create_log():
+    """
+    Create a log for the model completion using BackgroundTasks to create the log in the background
+    """
+    formatted_prompt = chevron.render(
+        PROMPT, {"context": RETRIEVED_DOCUMENTS, "question": QUESTION, "rules": RULES}
+    )
+
+    response = client.chat.completions.create(
+        messages=[
+            {
+                "role": "user",
+                "content": formatted_prompt,
+            }
+        ],
+        model="gpt-4o",
+    )
+
+    model_output = response.choices[0].message.content
+
+    ########################################################
+    # Example implementation of creating a non-blocking log event
+    ########################################################
+    quotient_logger.log(
+        model_input=QUESTION,
+        model_output=model_output,
+        documents=RETRIEVED_DOCUMENTS,
+        contexts=RULES,
+    )
+
+    return {"response": model_output}

examples/logging/example_fast_api/log.py

@@ -0,0 +1,6 @@
+from fastapi import FastAPI
+from log import router as log_router
+
+app = FastAPI()
+
+app.include_router(log_router)

examples/logging/example_fast_api/main.py

@@ -1,8 +1,9 @@
 [tool.poetry]
 name = "quotientai"
-version = "0.1.3"
+version = "0.1.4"
 authors = [
     "Freddie Vargus <[email protected]>",
+    "Michael Goitia Sarmiento <[email protected]>",
 ]
 description = "CLI for evaluating large language models with Quotient"
 readme = "README.md"

poetry.lock

@@ -1,5 +1,5 @@
 import os
-from typing import List
+from typing import Any, Dict, List, Optional
 
 import httpx
 
@@ -11,7 +11,6 @@
 from quotientai.resources.runs import Run
 
 
-
 class _BaseQuotientClient(httpx.Client):
     def __init__(self, api_key: str):
         super().__init__(
@@ -25,7 +24,7 @@ def _get(self, path: str) -> dict:
         return response
 
     @handle_errors
-    def _post(self, path: str, data: dict = {}) -> dict:
+    def _post(self, path: str, data: dict = {}, timeout: int = None) -> dict:
         if isinstance(data, dict):
             data = {k: v for k, v in data.items() if v is not None}
         elif isinstance(data, list):
@@ -34,6 +33,7 @@ def _post(self, path: str, data: dict = {}) -> dict:
         response = self.post(
             url=path,
             json=data,
+            timeout=timeout,
         )
         return response
 
@@ -52,6 +52,93 @@ def _delete(self, path: str) -> dict:
         return response
 
 
+class QuotientLogger:
+    """
+    Logger interface that wraps the underlying logs resource.
+    This class handles both configuration (via init) and logging.
+    """
+
+    def __init__(self, logs_resource):
+        self.logs_resource = logs_resource
+
+        self.app_name: Optional[str] = None
+        self.environment: Optional[str] = None
+        self.tags: Dict[str, Any] = {}
+        self.hallucination_detection: bool = False
+        self.inconsistency_detection: bool = False
+        self._configured = False
+
+    def init(
+        self,
+        *,
+        app_name: str,
+        environment: str,
+        tags: Optional[Dict[str, Any]] = {},
+        hallucination_detection: bool = False,
+        inconsistency_detection: bool = False,
+    ) -> "QuotientLogger":
+        """
+        Configure the logger with the provided parameters and return self.
+        This method must be called before using log().
+        """
+        self.app_name = app_name
+        self.environment = environment
+        self.tags = tags or {}
+        self.hallucination_detection = hallucination_detection
+        self.inconsistency_detection = inconsistency_detection
+        self._configured = True
+        return self
+
+    def log(
+        self,
+        *,
+        model_input: str,
+        model_output: str,
+        documents: List[dict],
+        contexts: Optional[List[str]] = None,
+        tags: Optional[Dict[str, Any]] = {},
+        hallucination_detection: Optional[bool] = None,
+        inconsistency_detection: Optional[bool] = None,
+    ):
+        """
+        Log the model interaction asynchronously.
+
+        Merges the default tags (set via init) with any runtime-supplied tags and calls the
+        underlying non_blocking_create function.
+        """
+        if not self._configured:
+            raise RuntimeError(
+                "Logger is not configured. Please call init() before logging."
+            )
+
+        # Merge default tags with any tags provided at log time.
+        merged_tags = {**self.tags, **(tags or {})}
+
+        # Use the instance variable as the default if not provided
+        hallucination_detection = (
+            hallucination_detection
+            if hallucination_detection is not None
+            else self.hallucination_detection
+        )
+        inconsistency_detection = (
+            inconsistency_detection
+            if inconsistency_detection is not None
+            else self.inconsistency_detection
+        )
+
+        return self.logs_resource.non_blocking_create(
+            app_name=self.app_name,
+            environment=self.environment,
+            model_input=model_input,
+            model_output=model_output,
+            documents=documents,
+            contexts=contexts,
+            tags=merged_tags,
+            hallucination_detection=hallucination_detection,
+            inconsistency_detection=inconsistency_detection,
+        )
+
+
 class QuotientAI:
     """
     A client that provides access to the QuotientAI API.
@@ -77,7 +164,10 @@ def __init__(self):
         self.models = resources.ModelsResource(_client)
         self.runs = resources.RunsResource(_client)
         self.metrics = resources.MetricsResource(_client)
+        self.logs = resources.LogsResource(_client)
 
+        # Create an unconfigured logger instance.
+        self.logger = QuotientLogger(self.logs)
 
     def evaluate(
         self,

pyproject.toml

@@ -3,11 +3,13 @@
 from quotientai.resources.datasets import DatasetsResource
 from quotientai.resources.runs import RunsResource
 from quotientai.resources.metrics import MetricsResource
+from quotientai.resources.logs import LogsResource
 
 __all__ = [
     "PromptsResource",
     "ModelsResource",
     "DatasetsResource",
     "RunsResource",
     "MetricsResource",
+    "LogsResource",
 ]