Merge pull request #37 from instructkr/dev-v2

LogicKor V2 Update

.github/workflows/pre-commit.yaml

@@ -0,0 +1,33 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: Get pip cache dir
+        id: pip-cache
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+      - name: pip/pre-commit cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ${{ steps.pip-cache.outputs.dir }}
+            ~/.cache/pre-commit
+          key: ${{ runner.os }}-pip-pre-commit-${{ hashFiles('**/.pre-commit-config.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-pre-commit
+      - name: pre-commit #  don't use in self-hosted `- uses: pre-commit/[email protected]`
+        run: |
+          pip install pre-commit
+          pre-commit install --install-hooks
+          pre-commit run -a

.gitignore

@@ -0,0 +1,4 @@
+.DS_Store
+.ipynb_checkpoints
+**/.DS_Store
+**/.ipynb_checkpoints

.pre-commit-config.yaml

@@ -0,0 +1,33 @@
+exclude: ^(exps)
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+        types: [file, python]
+      - id: trailing-whitespace
+        types: [file, python]
+      - id: mixed-line-ending
+      - id: check-added-large-files
+        args: [--maxkb=4096]
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.4.10
+    hooks:
+      # Run the linter.
+      - id: ruff
+      # Run the formatter.
+      - id: ruff-format
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        name: isort (python)
+        args: ["--profile", "black", "-l", "120", "--lines-after-imports", "2"]
+  - repo: https://github.com/pycqa/flake8.git
+    rev: 7.1.0
+    hooks:
+      - id: flake8
+        types: [python]
+        args: ["--max-line-length", "120", "--ignore", "F811,F841,E203,E402,E501,E712,W503,E704,E731"]

evaluator.py

@@ -0,0 +1,168 @@
+import argparse
+import json
+import re
+import time
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
+from pathlib import Path
+from threading import Lock
+from typing import Dict, Union
+
+import pandas as pd
+from openai import OpenAI
+
+from templates import JUDGE_TEMPLATE
+
+
+# Constants
+TIME_START = datetime.now().strftime("%Y%m%d_%H%M%S")
+LOCK = Lock()
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-o", "--model-output-dir", help="Model Output Directory", required=True
+    )
+    parser.add_argument("-k", "--openai-api-key", help="OpenAI API Key", required=True)
+    parser.add_argument(
+        "-j", "--judge-model", help="Judge Model", default="gpt-4-1106-preview"
+    )
+    parser.add_argument("-t", "--threads", help="Thread count", default=42, type=int)
+    return parser.parse_args()
+
+
+def create_azure_client(api_key: str):
+    return OpenAI(api_key=api_key)
+
+
+def create_answers(
+    client, model_output, judge_model, is_multi_turn: bool = False, i=0
+) -> Dict[str, Union[str, float]]:
+    model_questions = model_output["questions"]
+    model_outputs = model_output["outputs"]
+    model_references = model_output["references"]
+
+    prompt = (
+        f"아래의 내용을 주어진 평가 기준들을 충실히 반영하여 평가해라. 특히 모델 답변이 언어 요구사항을 준수하는지 반드시 확인해야 한다.\n\n"
+        f"**Question**\n{model_questions[0]}"
+    )
+
+    if model_references and model_references[0]:
+        prompt += f"\n\n**Additional Reference**\n{model_references[0]}"
+
+    prompt += f"\n\n**Model's Response**\n{model_outputs[0]}"
+
+    if is_multi_turn:
+        prompt += f"\n\n**Follow-up Question.**\n{model_questions[1]}"
+        if model_references and model_references[1]:
+            prompt += f"\n\n**Additional Reference**\n{model_references[1]}"
+        prompt += f"\n\n**Model's Response**\n{model_outputs[1]}"
+
+    prompt += "\n\n[[대화 종료. 평가 시작.]]"
+
+    try:
+        response = client.chat.completions.create(
+            model=judge_model,
+            temperature=0.0,
+            n=1,
+            messages=[
+                {
+                    "role": "system",
+                    "content": JUDGE_TEMPLATE[
+                        "multi_turn" if is_multi_turn else "single_turn"
+                    ],
+                },
+                {"role": "user", "content": prompt},
+            ],
+        )
+
+        content = response.choices[0].message.content
+        judge_message_match = re.search(
+            r"평가:(.*?)점수:", content.replace("*", ""), re.DOTALL
+        )
+        judge_message = (
+            judge_message_match.group(1).strip()
+            if judge_message_match
+            else "No judge message found"
+        )
+        judge_score_match = re.search(
+            r"점수:\s*(\d+(\.\d+)?)", content.replace("*", "")
+        )
+        if judge_score_match:
+            judge_score = float(judge_score_match.group(1))
+        else:
+            raise ValueError("No score found in response")
+
+        return {"judge_message": judge_message, "judge_score": judge_score}
+
+    except Exception as e:
+        print("Error. Retrying after 20 sec", e)
+        time.sleep(20)
+
+        # 꼭 아래 이유가 아닐 수 있음. 핸들링 필요.
+        if i > 3:
+            print("Impossible prompt, aborting..!")
+            return {
+                "judge_message": "Impossible to judge due to repetition.",
+                "judge_score": 0.0,
+            }
+        i += 1
+        return create_answers(client, model_output, judge_model, is_multi_turn, i)
+
+
+def process_item(client, row, judge_model, output_file):
+    query_single = create_answers(client, row, judge_model)
+    query_multi = create_answers(client, row, judge_model, is_multi_turn=True)
+
+    row["query_single"] = query_single
+    row["query_multi"] = query_multi
+    row = row.to_dict()
+
+    with LOCK:
+        with output_file.open("a", encoding="utf-8-sig") as f:
+            f.write(json.dumps(row, ensure_ascii=False))
+            f.write("\n")
+
+
+def process_file(
+    client, file_path: Path, output_dir: Path, judge_model, threads: int, args
+):
+    print(f"- 현재 Processing : {file_path}")
+    df_model_outputs = pd.read_json(file_path, lines=True)
+
+    output_file = output_dir / file_path.relative_to(args.model_output_dir)
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+
+    with ThreadPoolExecutor(max_workers=threads) as executor:
+        for row in df_model_outputs.iterrows():
+            executor.submit(process_item, client, row[1], judge_model, output_file)
+
+
+def is_hidden(filepath: Path) -> bool:
+    return any(part.startswith(".") for part in filepath.parts)
+
+
+def main():
+    args = get_args()
+    client = create_azure_client(args.openai_api_key)
+
+    input_dir = Path(args.model_output_dir)
+    output_dir = Path("./evaluated")
+
+    # Filter out hidden files
+    json_files = [file for file in input_dir.rglob("*.jsonl") if not is_hidden(file)]
+
+    for file_path in json_files:
+        output_file_path = output_dir / file_path.relative_to(input_dir)
+        if output_file_path.exists():
+            print(f"이미 평가 완료.. : {file_path}")
+            continue
+        process_file(
+            client, file_path, output_dir, args.judge_model, args.threads, args
+        )
+        time.sleep(20)  # to handle ratelimit!
+
+
+if __name__ == "__main__":
+    main()