refactor: streamline evaluator and validator logic to require only GEMINI_API_KEY; remove OpenAI fallback

feat: enhance evaluation process by integrating full video input for Gemini
test: update tests to reflect changes in evaluation logic and ensure compatibility with new video input
chore: update dependencies in uv.lock

Author: vex0209-bt

env.example

@@ -8,14 +8,13 @@
 # Centralized API URL (validators use for GET /tasks/latest, GET /weights, POST evaluations)
 API_URL=https://api.leoma.ai
 
-# Validator evaluator LLM keys. At least one must be set.
-# - GEMINI_API_KEY (preferred, cheaper): gemini-3.1-flash-lite-preview
-# - OPENAI_API_KEY (fallback): gpt-4o
-# If both are set, Gemini is used first and GPT-4o is the per-request fallback
-# when Gemini fails.
+# Validator evaluator LLM key. REQUIRED for validators.
+# - GEMINI_API_KEY: gemini-3.1-flash-lite-preview. The evaluator sends the full
+#   generated video to Gemini (no GPT-4o fallback).
 # Gemini key: https://aistudio.google.com/app/apikey
-# OpenAI key: https://platform.openai.com/api-keys
 GEMINI_API_KEY=
+# OPENAI_API_KEY: NOT used by validators. Required only by the owner-sampler
+# (gpt-4o benchmark prompt/description generation). See OWNER-SAMPLER section.
 OPENAI_API_KEY=
 
 # ═══════════════════════════════════════════════════════════════════════════════

leoma/app/evaluator/main.py

@@ -18,12 +18,10 @@
 from typing import Set
 
 from google import genai
-from openai import AsyncOpenAI
 
 from leoma.bootstrap import (
     GEMINI_API_KEY,
     OBJECT_STORAGE_BACKEND,
-    OPENAI_API_KEY,
     SAMPLES_BUCKET,
 )
 from leoma.bootstrap import emit_log as log, emit_header as log_header, log_exception
@@ -32,14 +30,12 @@
     list_evaluated_task_ids,
     download_task_artifacts,
 )
-from leoma.infra.video_utils import extract_frames, frames_to_base64
+from leoma.infra.video_utils import frames_to_base64
 from leoma.infra.judge import evaluate_generated_video_async
 
 EVALUATOR_POLL_INTERVAL = int(os.environ.get("EVALUATOR_POLL_INTERVAL", "60"))
 EVALUATED_LIST_MAX = 100
 API_URL = os.environ.get("API_URL", "https://api.leoma.ai")
-EVALUATION_MAX_FRAMES = int(os.environ.get("EVALUATION_MAX_FRAMES", "12"))
-EVALUATION_FRAME_FPS = float(os.environ.get("EVALUATION_FRAME_FPS", "3"))
 EVALUATION_PASS_THRESHOLD = int(os.environ.get("EVALUATION_PASS_THRESHOLD", "75"))
 EVALUATION_CRITICAL_THRESHOLD = int(os.environ.get("EVALUATION_CRITICAL_THRESHOLD", "50"))
 
@@ -72,18 +68,11 @@ async def run_evaluator_loop() -> None:
     from leoma.bootstrap import WALLET_NAME, HOTKEY_NAME
 
     gemini_key = GEMINI_API_KEY or os.environ.get("GEMINI_API_KEY")
-    openai_key = OPENAI_API_KEY or os.environ.get("OPENAI_API_KEY")
-    if not gemini_key and not openai_key:
-        log("Evaluator requires GEMINI_API_KEY and/or OPENAI_API_KEY (fallback)", "error")
+    if not gemini_key:
+        log("Evaluator requires GEMINI_API_KEY", "error")
         return
-    gemini_client = genai.Client(api_key=gemini_key) if gemini_key else None
-    openai_client = AsyncOpenAI(api_key=openai_key) if openai_key else None
-    if gemini_client and openai_client:
-        log("Evaluator LLM: Gemini primary, GPT-4o fallback", "info")
-    elif gemini_client:
-        log("Evaluator LLM: Gemini only (no GPT-4o fallback configured)", "info")
-    else:
-        log("Evaluator LLM: GPT-4o only (GEMINI_API_KEY not set)", "info")
+    gemini_client = genai.Client(api_key=gemini_key)
+    log("Evaluator LLM: Gemini only (full-video evaluation)", "info")
     try:
         s3_client = create_samples_read_client()
     except ValueError as e:
@@ -182,22 +171,13 @@ async def run_evaluator_loop() -> None:
             samples_payload: list = []
             eval_entries: list = []
             for miner_hotkey, gen_video_path in generated_videos.items():
-                gen_frames_dir = os.path.join(dest_dir, f"gen_frames_{miner_hotkey[:8]}")
                 try:
                     latency_ms = miner_latencies_ms.get(miner_hotkey)
-                    gen_frames = await extract_frames(
-                        gen_video_path,
-                        gen_frames_dir,
-                        max_frames=EVALUATION_MAX_FRAMES,
-                        fps=EVALUATION_FRAME_FPS,
-                    )
-                    gen_frames_b64 = frames_to_base64(gen_frames)
                     comparison = await evaluate_generated_video_async(
                         first_frame_b64,
-                        gen_frames_b64,
+                        gen_video_path,
                         description,
                         gemini_client=gemini_client,
-                        openai_client=openai_client,
                         pass_threshold=EVALUATION_PASS_THRESHOLD,
                         critical_threshold=EVALUATION_CRITICAL_THRESHOLD,
                     )
@@ -248,9 +228,6 @@ async def run_evaluator_loop() -> None:
                     log(f"Miner {miner_hotkey[:12]}...: {'PASSED' if passed else 'FAILED'}", "info")
                 except Exception as e:
                     log(f"Miner {miner_hotkey[:12]}... evaluation failed: {e}", "error")
-                finally:
-                    if os.path.exists(gen_frames_dir):
-                        _remove_directory(gen_frames_dir)
 
             if samples_payload:
                 signature = api_client.sign_evaluation_payload(eval_entries)

leoma/app/validator/main.py

@@ -18,7 +18,6 @@
     EPOCH_LEN,
     OBJECT_STORAGE_BACKEND,
     GEMINI_API_KEY,
-    OPENAI_API_KEY,
     WALLET_NAME,
     HOTKEY_NAME,
     NETWORK,
@@ -113,8 +112,8 @@ async def main() -> None:
     """Main entry point: run evaluator (background) + weight-setting loop."""
     log_header("Leoma Validator Starting (evaluator + weight-setter)")
 
-    if not GEMINI_API_KEY and not OPENAI_API_KEY:
-        log("Evaluator requires GEMINI_API_KEY and/or OPENAI_API_KEY (fallback)", "error")
+    if not GEMINI_API_KEY:
+        log("Evaluator requires GEMINI_API_KEY", "error")
         return
 
     log(f"Using centralized API: {API_URL}", "info")

leoma/infra/judge.py

@@ -5,6 +5,7 @@
 import asyncio
 import base64
 import json
+import os
 from typing import Any, Dict, List
 from leoma.bootstrap import emit_log as log
 from openai import AsyncOpenAI
@@ -15,6 +16,17 @@
 GEMINI_EVAL_MAX_ATTEMPTS = 3
 GEMINI_EVAL_RETRY_SLEEP_S = 300
 
+GEMINI_EVAL_VIDEO_FPS = float(os.environ.get("EVALUATION_VIDEO_FPS", "16"))
+GEMINI_EVAL_MEDIA_RESOLUTION = os.environ.get(
+    "EVALUATION_MEDIA_RESOLUTION", "high"
+).strip().lower()
+
+_MEDIA_RESOLUTION_MAP = {
+    "low": genai_types.MediaResolution.MEDIA_RESOLUTION_LOW,
+    "medium": genai_types.MediaResolution.MEDIA_RESOLUTION_MEDIUM,
+    "high": genai_types.MediaResolution.MEDIA_RESOLUTION_HIGH,
+}
+
 DESCRIPTION_PROMPT = """You are writing a benchmark prompt for first-frame-conditioned video generation.
 
 Use the provided sequential frames from a 5-second one-shot clip.
@@ -162,7 +174,7 @@ def _build_eval_instructions(prompt: str) -> str:
 Given:
 1) A conditioning first frame
 2) The benchmark generation prompt
-3) Sequential frames from a generated video
+3) The full generated video
 
 Benchmark prompt:
 "{prompt}"
@@ -211,19 +223,29 @@ def _build_eval_instructions(prompt: str) -> str:
 )
 
 
+def _video_to_gemini_part(video_path: str) -> genai_types.Part:
+    """Inline the full generated video, with explicit fps so Gemini samples it
+    densely instead of its 1 fps default."""
+    with open(video_path, "rb") as f:
+        data = f.read()
+    return genai_types.Part(
+        inline_data=genai_types.Blob(data=data, mime_type="video/mp4"),
+        video_metadata=genai_types.VideoMetadata(fps=GEMINI_EVAL_VIDEO_FPS),
+    )
+
+
 async def _evaluate_via_gemini(
     gemini_client: genai.Client,
     first_frame: List[Dict[str, Any]],
-    generated_frames: List[Dict[str, Any]],
+    generated_video_path: str,
     prompt: str,
 ) -> str:
     first_frame_parts = [_frame_dict_to_gemini_part(f) for f in first_frame]
-    gen_frame_parts = [_frame_dict_to_gemini_part(f) for f in generated_frames]
 
     contents: List[Any] = [_build_eval_instructions(prompt), "CONDITIONING FIRST FRAME:"]
     contents.extend(first_frame_parts)
-    contents.append("GENERATED VIDEO FRAMES (chronological):")
-    contents.extend(gen_frame_parts)
+    contents.append("GENERATED VIDEO:")
+    contents.append(_video_to_gemini_part(generated_video_path))
 
     response = await gemini_client.aio.models.generate_content(
         model=GEMINI_EVAL_MODEL,
@@ -233,79 +255,48 @@ async def _evaluate_via_gemini(
             temperature=0.1,
             max_output_tokens=450,
             response_mime_type="application/json",
+            media_resolution=_MEDIA_RESOLUTION_MAP.get(
+                GEMINI_EVAL_MEDIA_RESOLUTION,
+                genai_types.MediaResolution.MEDIA_RESOLUTION_HIGH,
+            ),
         ),
     )
     return response.text or ""
 
 
-async def _evaluate_via_openai(
-    openai_client: AsyncOpenAI,
-    first_frame: List[Dict[str, Any]],
-    generated_frames: List[Dict[str, Any]],
-    prompt: str,
-) -> str:
-    content = [
-        {"type": "text", "text": _build_eval_instructions(prompt)},
-        {"type": "text", "text": "CONDITIONING FIRST FRAME:"},
-    ] + first_frame + [
-        {"type": "text", "text": "GENERATED VIDEO FRAMES (chronological):"},
-    ] + generated_frames
-
-    response = await openai_client.chat.completions.create(
-        model="gpt-4o",
-        messages=[
-            {"role": "system", "content": EVAL_SYSTEM_MSG},
-            {"role": "user", "content": content},
-        ],
-        max_tokens=450,
-    )
-    return response.choices[0].message.content or ""
-
-
 async def evaluate_generated_video_async(
     first_frame: List[Dict[str, Any]],
-    generated_frames: List[Dict[str, Any]],
+    generated_video_path: str,
     prompt: str,
     *,
     gemini_client: genai.Client | None = None,
-    openai_client: AsyncOpenAI | None = None,
     pass_threshold: int = 70,
     critical_threshold: int = 50,
 ) -> Dict[str, Any]:
-    if gemini_client is None and openai_client is None:
-        raise ValueError("evaluate_generated_video_async requires gemini_client or openai_client")
+    if gemini_client is None:
+        raise ValueError("evaluate_generated_video_async requires gemini_client")
 
     raw: str | None = None
-    if gemini_client is not None:
-        last_error: Exception | None = None
-        for attempt in range(1, GEMINI_EVAL_MAX_ATTEMPTS + 1):
-            try:
-                raw = await _evaluate_via_gemini(
-                    gemini_client, first_frame, generated_frames, prompt
-                )
-                break
-            except Exception as e:
-                last_error = e
-                log(
-                    f"Gemini evaluation attempt {attempt}/{GEMINI_EVAL_MAX_ATTEMPTS} "
-                    f"failed: {e}",
-                    "warn",
-                )
-                if attempt < GEMINI_EVAL_MAX_ATTEMPTS:
-                    await asyncio.sleep(GEMINI_EVAL_RETRY_SLEEP_S)
-
-        if raw is None:
-            if openai_client is None:
-                assert last_error is not None
-                raise last_error
+    last_error: Exception | None = None
+    for attempt in range(1, GEMINI_EVAL_MAX_ATTEMPTS + 1):
+        try:
+            raw = await _evaluate_via_gemini(
+                gemini_client, first_frame, generated_video_path, prompt
+            )
+            break
+        except Exception as e:
+            last_error = e
             log(
-                f"Gemini evaluation failed after {GEMINI_EVAL_MAX_ATTEMPTS} attempts; "
-                "falling back to GPT-4o",
+                f"Gemini evaluation attempt {attempt}/{GEMINI_EVAL_MAX_ATTEMPTS} "
+                f"failed: {e}",
                 "warn",
             )
+            if attempt < GEMINI_EVAL_MAX_ATTEMPTS:
+                await asyncio.sleep(GEMINI_EVAL_RETRY_SLEEP_S)
 
     if raw is None:
-        raw = await _evaluate_via_openai(openai_client, first_frame, generated_frames, prompt)
+        assert last_error is not None
+        raise last_error
 
     text = _strip_json_fence(raw)
     try: