feat(evaluation): add auditable quality and cost workflow

.env.example

@@ -183,6 +183,18 @@ REDIS_PREFIX=stream:
 # 后台任务每模型并发上限（0/负数禁用；交互 chat 不受其限制）
 # WEKNORA_MODEL_MAX_CONCURRENCY=32
 
+# --- 课题三：模型调用可观测性（默认启用）---
+# 只记录租户、模型、用途、Token、缓存、费用、耗时和成功状态，不记录 Prompt/响应正文。
+# WEKNORA_MODEL_CALL_OBSERVABILITY_ENABLED=true
+# 普通问答、评测和 Wiki 模型调用元数据保留天数，默认 30 天。
+# WEKNORA_MODEL_CALL_RETENTION_DAYS=30
+# Prompt 稳定前缀的 HMAC 密钥。建议设置为独立随机密钥；未设置时不保存前缀指纹，避免弱哈希泄露。
+# 不要复用 JWT、数据库或模型 API 密钥，也不要把真实值提交到 Git。
+# WEKNORA_MODEL_CALL_FINGERPRINT_KEY=replace-with-a-random-secret
+# 评测数据集根目录。default 映射到 <根目录>/samples，其他 dataset_id 映射到 <根目录>/<dataset_id>。
+# 每个目录必须包含 manifest.json 和五个 Parquet 文件；默认 ./dataset。
+# WEKNORA_EVALUATION_DATASET_DIR=./dataset
+
 # ========== B3. 文件存储（通用）==========
 # 文件存储类型：local / minio / cos / tos / s3 / obs / oss / dummy。
 STORAGE_TYPE=local

.github/workflows/evaluation-regression.yml

@@ -0,0 +1,161 @@
+name: Evaluation regression
+
+on:
+  pull_request:
+    paths:
+      - 'cmd/evalgate/**'
+      - 'cmd/cachebench/**'
+      - 'cmd/evidenceverify/**'
+      - 'cmd/retrievalgate/**'
+      - 'evaluation/**'
+      - 'frontend/src/api/evaluation/**'
+      - 'frontend/src/api/model/modelUsage*'
+      - 'frontend/src/components/ModelEditorDialog.vue'
+      - 'frontend/src/utils/evaluation*'
+      - 'frontend/src/views/settings/EvaluationSettings*'
+      - 'frontend/src/views/settings/ModelSettings.vue'
+      - 'frontend/src/views/settings/Settings.vue'
+      - 'frontend/src/i18n/locales/**'
+      - 'frontend/package.json'
+      - 'frontend/package-lock.json'
+      - 'internal/config/config.go'
+      - 'internal/config/model_observability_test.go'
+      - 'internal/handler/evaluation*'
+      - 'internal/application/service/evaluation*'
+      - 'internal/application/service/model_call_recorder*'
+      - 'internal/application/service/embedding_cache_persistence*'
+      - 'internal/application/service/metric/**'
+      - 'internal/application/service/chat_pipeline/**'
+      - 'internal/application/service/knowledgebase_search*.go'
+      - 'internal/models/chat/**'
+      - 'internal/models/embedding/**'
+      - 'internal/models/rerank/**'
+      - 'internal/types/evaluation.go'
+      - 'internal/types/interfaces/evaluation.go'
+      - 'internal/types/llm_observability.go'
+      - 'migrations/sqlite/000017_evaluation_tasks*'
+      - 'migrations/sqlite/000018_evaluation_model_calls*'
+      - 'migrations/sqlite/000019_evaluation_model_call_cost*'
+      - 'migrations/sqlite/000020_global_model_call_observability*'
+      - 'migrations/sqlite/000021_evaluation_run_config*'
+      - 'migrations/sqlite/000022_model_call_types*'
+      - 'migrations/sqlite/000023_embedding_cache_entries*'
+      - 'migrations/sqlite/000024_evaluation_usage_snapshot*'
+      - 'migrations/sqlite/000025_embedding_cache_usage*'
+      - 'migrations/sqlite/000026_model_call_request_fingerprint*'
+      - 'migrations/sqlite/000027_evaluation_task_heartbeat*'
+      - 'migrations/versioned/000096_evaluation_tasks*'
+      - 'migrations/versioned/000097_evaluation_model_calls*'
+      - 'migrations/versioned/000098_evaluation_model_call_cost*'
+      - 'migrations/versioned/000099_global_model_call_observability*'
+      - 'migrations/versioned/000100_evaluation_run_config*'
+      - 'migrations/versioned/000101_model_call_types*'
+      - 'migrations/versioned/000102_embedding_cache_entries*'
+      - 'migrations/versioned/000103_evaluation_usage_snapshot*'
+      - 'migrations/versioned/000104_embedding_cache_usage*'
+      - 'migrations/versioned/000105_model_call_request_fingerprint*'
+      - 'migrations/versioned/000106_evaluation_task_heartbeat*'
+      - '.env.example'
+      - 'Makefile'
+      - '.github/workflows/evaluation-regression.yml'
+  schedule:
+    - cron: '0 18 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  deterministic-gate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+      - name: Test evaluation metrics and executable retrieval gate
+        run: >-
+          go test ./internal/application/service/metric
+          ./internal/application/service/chat_pipeline
+          ./internal/application/service
+          ./internal/config
+          ./internal/models/chat
+          ./internal/models/embedding
+          ./cmd/retrievalgate ./cmd/evalgate ./cmd/cachebench
+      - name: Generate and gate production-pipeline result
+        run: go run ./cmd/retrievalgate -out evaluation-pipeline-result.json
+      - name: Verify generated production-pipeline result
+        run: >-
+          go run ./cmd/evalgate
+          -result evaluation-pipeline-result.json
+          -reference-result evaluation/fixtures/regression_reference.json
+          -baseline evaluation/baseline.json
+          -report evaluation-report.json
+      - name: Prove a recall regression is rejected and identified
+        shell: bash
+        run: |
+          if go run ./cmd/evalgate \
+            -result evaluation/fixtures/regression_degraded.json \
+            -reference-result evaluation/fixtures/regression_reference.json \
+            -baseline evaluation/baseline.json \
+            -report evaluation-degraded-report.json; then
+            echo "expected the degraded evaluation to fail" >&2
+            exit 1
+          fi
+          grep -q '"path": "metric.retrieval_metrics.recall"' evaluation-degraded-report.json
+          grep -A8 '"path": "metric.retrieval_metrics.recall"' evaluation-degraded-report.json | grep -q '"passed": false'
+      - name: Upload regression report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: evaluation-regression-report
+          path: evaluation-report.json
+          if-no-files-found: warn
+      - name: Upload degradation proof
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: evaluation-degradation-proof
+          path: evaluation-degraded-report.json
+          if-no-files-found: warn
+      - name: Upload production-pipeline result
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: evaluation-production-pipeline-result
+          path: evaluation-pipeline-result.json
+          if-no-files-found: error
+      - name: Verify cache comparison example
+        run: >-
+          go run ./cmd/cachebench
+          -strict
+          -before evaluation/fixtures/cache_before.json
+          -after evaluation/fixtures/cache_after.json
+          -report cache-comparison.json
+      - name: Upload cache comparison report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cache-comparison-report
+          path: cache-comparison.json
+          if-no-files-found: warn
+
+  evaluation-ui:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: npm
+          cache-dependency-path: frontend/package-lock.json
+      - name: Install frontend dependencies
+        working-directory: frontend
+        run: npm ci
+      - name: Test evaluation and model usage UI
+        working-directory: frontend
+        run: npm test
+      - name: Type-check and build evaluation UI
+        working-directory: frontend
+        run: npm run build-with-types

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help build run test clean docker-build-app docker-build-docreader docker-build-frontend docker-build-all docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama build-images build-images-app build-images-docreader build-images-frontend clean-images check-env list-containers pull-images show-platform dev-start dev-stop dev-restart dev-logs dev-status dev-app dev-frontend docs install-swagger build-lite run-lite package-lite anydoc-lib build-anydoc
+.PHONY: help build run test evaluation-gate clean docker-build-app docker-build-docreader docker-build-frontend docker-build-all docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama build-images build-images-app build-images-docreader build-images-frontend clean-images check-env list-containers pull-images show-platform dev-start dev-stop dev-restart dev-logs dev-status dev-app dev-frontend docs install-swagger build-lite run-lite package-lite anydoc-lib build-anydoc
 
 # Show help
 help:
@@ -8,6 +8,7 @@ help:
 	@echo "  build             构建应用"
 	@echo "  run               运行应用"
 	@echo "  test              运行测试"
+	@echo "  evaluation-gate   一条命令运行课题三质量门禁并生成评测/缓存报告"
 	@echo "  anydoc-lib        构建 anydoc 静态库（需要 Rust 工具链）"
 	@echo "  build-anydoc      构建带 anydoc 解析引擎的应用"
 	@echo "  clean             清理构建文件"
@@ -107,6 +108,14 @@ run: build
 test:
 	go test -v ./...
 
+# Topic 3 deterministic acceptance command. Reports are written to the current
+# directory so the command output and submitted artifacts use identical data.
+evaluation-gate:
+	go test ./internal/application/service/metric ./internal/application/service/chat_pipeline ./internal/application/service ./internal/config ./internal/models/chat ./internal/models/embedding ./cmd/retrievalgate ./cmd/evalgate ./cmd/cachebench
+	go run ./cmd/retrievalgate -out evaluation-pipeline-result.json
+	go run ./cmd/evalgate -result evaluation-pipeline-result.json -reference-result evaluation/fixtures/regression_reference.json -baseline evaluation/baseline.json -report evaluation-report.json
+	go run ./cmd/cachebench -strict -before evaluation/fixtures/cache_before.json -after evaluation/fixtures/cache_after.json -report cache-comparison.json
+
 # Clean build artifacts
 clean:
 	go clean
@@ -348,5 +357,3 @@ dev-app:
 
 dev-frontend:
 	./scripts/dev.sh frontend
-
-