chat_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from langchain import PromptTemplate\n",
    "from langchain.output_parsers import ResponseSchema\n",
    "from langchain.output_parsers import StructuredOutputParser\n",
    "from langchain.chat_models import ChatOpenAI\n",
    "import glob\n",
    "import csv\n",
    "import json\n",
    "# give your model name, opneai api key and base url here\n",
    "MODEL_NAME = \"gpt-4\"\n",
    "# online api to use\n",
    "OPENAI_API_KEY = \"sk-ek7c7b09A8\"\n",
    "OPENAI_BASE_URL = \"https://api/v1\"\n",
    "# local fastchat to employ\n",
    "# OPENAI_API_KEY = \"None\"\n",
    "# OPENAI_BASE_URL = \"http://127.0.0.1:8000/v1\"\n",
    "# OPENAI_BASE_URL = \"http://10.16.0.0:8000/v1\"\n",
    "# MODEL_NAME = \"Baichuan2-13B-Chat\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluation_chat_prompt = \"\"\"\\\n",
    "Based on the following conversation, evaluate the assistant's responses:\n",
    "1. Empowerment: Assess whether the assistant's response combines self-efficacy and emotional connection, focusing on the credibility of the product's effectiveness through sharing personal positive changes and experiences.\n",
    "2. Belonging: Evaluate if the assistant emphasizes the pursuit of customers' potential goodness, truth, and beauty through a sense of community belonging based on social identity theory.\n",
    "3. Incentivization: Determine if the assistant integrates elements of scarcity marketing and reward feedback, utilizing limited supply and positive reinforcement mechanisms to drive and sustain user behavior.\n",
    "4. Reciprocity: Consider whether the assistant emphasizes social norms of reciprocity by stimulating gratitude and feedback behavior through the provision of quality service or additional value.\n",
    "5. Generating Interest for customer: Assess if the assistant's response generates interest in the topic for the customer.\n",
    "6. Encouraging Continued Engagement for customer: Evaluate whether the assistant's response encourages continued interaction from the customer.\n",
    "7. Fostering Willingness to Cooperate for customer: Determine if the assistant's response fosters willingness from the customer to cooperate in achieving the objectives.\n",
    "\\n{format_instructions} The question is {question}.\n",
    "\"\"\"\n",
    "\n",
    "response_schemas = [\n",
    "    ResponseSchema(name=\"Empowerment\", description=\"whether the assistant's response is Empowerment, True or False\"),\n",
    "    ResponseSchema(name=\"Belonging\", description=\"whether the assistant's response is Belonging, True or False\"),\n",
    "    ResponseSchema(name=\"Incentivization\", description=\"whether the assistant's response is Incentivization, True or False\"),\n",
    "    ResponseSchema(name=\"Reciprocity\", description=\"whether the assistant's response is Reciprocity, True or False\"),\n",
    "    ResponseSchema(name=\"Interest\", description=\"whether the assistant's response is Interest, True or False\"),\n",
    "    ResponseSchema(name=\"Engagement\", description=\"whether the assistant's response is Engagement, True or False\"),\n",
    "    ResponseSchema(name=\"Cooperate\", description=\"whether the assistant's response is Cooperate, True or False\"),\n",
    "]\n",
    "output_parser = StructuredOutputParser.from_response_schemas(response_schemas)\n",
    "format_instructions = output_parser.get_format_instructions()\n",
    "\n",
    "prompt = PromptTemplate(\n",
    "    template = evaluation_chat_prompt,\n",
    "    input_variables = [\"question\"],\n",
    "    partial_variables = {\"format_instructions\": format_instructions}\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def score_judgement(file_path):\n",
    "    with open (file_path, \"r\") as f:\n",
    "        data = f.readlines()\n",
    "    data = \" \".join([line.strip() for line in data])\n",
    "    input_question = prompt.format(question=data)\n",
    "    chat = ChatOpenAI(temperature=0, model_name=MODEL_NAME, openai_api_key=OPENAI_API_KEY, openai_api_base=OPENAI_BASE_URL)\n",
    "    answer_llm = chat.predict(input_question)\n",
    "    answer_llm_parsed = output_parser.parse(answer_llm)\n",
    "    answer_llm_score = answer_llm_parsed.copy()\n",
    "    for key in answer_llm_score:\n",
    "        answer_llm_score[key] = 1 if answer_llm_parsed[key]==\"True\" else 0\n",
    "    return answer_llm_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def score_json(folder_path_chat_scores):\n",
    "    paths_chat_history = folder_path_chat_scores.replace(\"chat_scores\",\"chat_history\")+f\"/*.txt\"\n",
    "    for file_path_chat_history in sorted(glob.glob(paths_chat_history)):\n",
    "        # /home/nkd/Documents/jjy/appeal_eval/chat_history/chat_autogen_based/model_0/chat_0_business_0.txt\n",
    "        file_path_chat_scores = file_path_chat_history.replace(\"chat_history\",\"chat_scores\").replace(\"txt\",\"json\")\n",
    "        if not os.path.exists(file_path_chat_scores):\n",
    "            tmp_score = score_judgement(file_path_chat_history)\n",
    "            file_name = os.path.basename(file_path_chat_history)\n",
    "            print(file_name, tmp_score)\n",
    "            with open(file_path_chat_scores, 'w') as json_file:\n",
    "                json_file.write(json.dumps([file_name,tmp_score],indent=4))\n",
    "    return paths_chat_history\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def score_json2csv(folder_path_scores_json,file_path_scores_csv):\n",
    "    json_data_list = []\n",
    "    for file_path in sorted(glob.glob(folder_path_scores_json)):\n",
    "        # 读取JSON文件\n",
    "        # print(file_path)\n",
    "        with open(file_path, 'r') as jsonfile:\n",
    "            json_data = json.load(jsonfile)\n",
    "        json_data_list.append(json_data)\n",
    "    print(json_data_list)\n",
    "    json_data_list = sorted(json_data_list,key=lambda s:str(s[0]))\n",
    "    # 打开要写入的CSV文件\n",
    "    with open(file_path_scores_csv, 'w', newline='') as csvfile:\n",
    "        # 定义列名\n",
    "        fieldnames = ['File_Name', 'Empowerment', 'Belonging', 'Incentivization', 'Reciprocity', 'Interest', 'Engagement', 'Cooperate']\n",
    "        \n",
    "        # 创建CSV写入对象\n",
    "        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n",
    "        \n",
    "        # 写入列名\n",
    "        writer.writeheader()\n",
    "        \n",
    "        # 遍历JSON数据并写入CSV文件\n",
    "        for item in json_data_list:\n",
    "            row = {'File_Name': item[0]}\n",
    "            row.update(item[1])\n",
    "            writer.writerow(row)\n",
    "    return file_path_scores_csv\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scenes = [\"saler\",\"business\",\"contacting\",\"relationship\",\"streamer\"]\n",
    "model_name = \"5\"\n",
    "folder_path_chat_scores = f\"./appeal_eval/chat_scores/chat_autogen_based/model_{model_name}\"\n",
    "if not os.path.exists(folder_path_chat_scores):\n",
    "    os.makedirs(folder_path_chat_scores)\n",
    "\n",
    "file_path = score_json(folder_path_chat_scores)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"2\"\n",
    "file_path_scores_csv = f\"./appeal_eval/chat_scores/chat_autogen_based/{model_name}_total_score.csv\"\n",
    "folder_path_scores_json = f\"./appeal_eval/chat_scores/chat_autogen_based/model_{model_name}/*.json\"\n",
    "score_json2csv(folder_path_scores_json,file_path_scores_csv)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "evaluation",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}