Construct_Dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "from docx import Document\n",
    "import os\n",
    "import openai\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "openai.api_key = os.getenv(\"OPENAI_GPT4_KEY\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_docx(file_path):\n",
    "    doc = Document(file_path)\n",
    "    full_text = []\n",
    "    for para in doc.paragraphs:\n",
    "        full_text.append(para.text)\n",
    "    return ' '.join(full_text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_sentences(text, n):\n",
    "    sentences = re.split(r'(?<=[^A-Z].[.?]) +(?=[A-Z])', text)\n",
    "    chunks = [sentences[i:i + n] for i in range(0, len(sentences), n)]\n",
    "    return chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_dataframe(file_path):\n",
    "    text = read_docx(file_path)\n",
    "    chunks = split_sentences(text, 4)\n",
    "    df = pd.DataFrame({'completion': [' '.join(chunk) for chunk in chunks]})\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_summaries(thoughts_df):\n",
    "    thoughts = thoughts_df['completion'].tolist()\n",
    "    summaries = []\n",
    "    for thought in tqdm(thoughts):\n",
    "        message=[\n",
    "            {\"role\": \"user\", \"content\": \"\"\"The following text enclosed within carrot brackets <> is a snippet of somebody's stream of consciousness. Please summarize what they are thinking about in one brief sentence written as an instruction that would cause the thought to be generated. For example, your sentence could be formatted as \"Think about ...\", format your response as just this sentence and nothing else.\\n\\n<\"\"\" + thought + \">\"}\n",
    "        ]\n",
    "        summary = openai.ChatCompletion.create(model=\"gpt-4\", messages=message)[\"choices\"][0][\"message\"][\"content\"]\n",
    "        summaries.append(summary)\n",
    "    return summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 109/109 [05:37<00:00,  3.09s/it]\n"
     ]
    }
   ],
   "source": [
    "file_path = 'Thoughts.docx'\n",
    "thoughts_df = create_dataframe(file_path)\n",
    "thoughts_df.insert(0, 'prompt', create_summaries(thoughts_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "thoughts_df.to_csv('Thoughts_Train_File.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# os.system(\"openai tools fine_tunes.prepare_data -f Thoughts_Train_File.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# openai api fine_tunes.create -t Thoughts_Train_File_prepared.jsonl -m davinci\n",
    "# openai api fine_tunes.get -i ft-b32V3ts3ipRPPEOcMjqKnvp6\n",
    "# openai api fine_tunes.follow -i ft-b32V3ts3ipRPPEOcMjqKnvp6"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}