learnsphere-transform.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['user_id', 'card_id', 'card_text', 'is_new_fact',\n",
      "       'user_n_study_positive', 'user_n_study_negative', 'user_n_study_total',\n",
      "       'card_n_study_positive', 'card_n_study_negative', 'card_n_study_total',\n",
      "       'usercard_n_study_positive', 'usercard_n_study_negative',\n",
      "       'usercard_n_study_total', 'acc_user', 'acc_card', 'acc_usercard',\n",
      "       'usercard_delta', 'usercard_delta_previous',\n",
      "       'usercard_previous_study_response', 'leitner_box', 'sm2_efactor',\n",
      "       'sm2_interval', 'sm2_repetition', 'delta_to_leitner_scheduled_date',\n",
      "       'delta_to_sm2_scheduled_date', 'repetition_model',\n",
      "       'elapsed_milliseconds', 'correct_on_first_try', 'utc_datetime',\n",
      "       'utc_date', 'response', 'record_id', 'deck_id', 'deck_name',\n",
      "       'n_minutes_spent'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "\n",
    "df = pd.read_hdf(\"./22-08-10-retention_features.h5\")\n",
    "print(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method NDFrame.head of        user_id card_id                                          card_text  \\\n",
       "0           46   26811  People of the steppes of central Asia, governe...   \n",
       "1           46   39163   Depose Hassan, the last Dey ruler of this city.    \n",
       "2           46   31494  The historian Bede was from, this kingdom cent...   \n",
       "3           46   22848  Kara Mustafa's attempts to take Vienna were th...   \n",
       "4           46   33260  Man overthrown by Francisco Pizarro, an Incan ...   \n",
       "...        ...     ...                                                ...   \n",
       "123138     404   21394  Titan whom Zeus bound to a rock for giving fir...   \n",
       "123139     404   38135  Running for President again in 2012, Russian l...   \n",
       "123140     404   30418  Identify this scandal named for an oil field i...   \n",
       "123141     404   34616        Position held by two people named Montezuma   \n",
       "123142     404   38453  Position of leadership by which Enrico Dandolo...   \n",
       "\n",
       "        is_new_fact  user_n_study_positive  user_n_study_negative  \\\n",
       "0              True                      0                      0   \n",
       "1              True                      1                      0   \n",
       "2              True                      1                      1   \n",
       "3              True                      1                      2   \n",
       "4              True                      2                      2   \n",
       "...             ...                    ...                    ...   \n",
       "123138         True                     32                     13   \n",
       "123139         True                     33                     13   \n",
       "123140         True                     34                     13   \n",
       "123141         True                     35                     13   \n",
       "123142         True                     35                     14   \n",
       "\n",
       "        user_n_study_total  card_n_study_positive  card_n_study_negative  \\\n",
       "0                        0                      0                      0   \n",
       "1                        1                      0                      0   \n",
       "2                        2                      0                      0   \n",
       "3                        3                      0                      0   \n",
       "4                        4                      0                      0   \n",
       "...                    ...                    ...                    ...   \n",
       "123138                  45                      0                      0   \n",
       "123139                  46                     19                      0   \n",
       "123140                  47                      0                      0   \n",
       "123141                  48                      0                      0   \n",
       "123142                  49                      1                      0   \n",
       "\n",
       "        card_n_study_total  ...  repetition_model  elapsed_milliseconds  \\\n",
       "0                        0  ...           karl100                  5000   \n",
       "1                        0  ...           karl100                  1000   \n",
       "2                        0  ...           karl100                  5000   \n",
       "3                        0  ...           karl100                  3000   \n",
       "4                        0  ...           karl100                  4000   \n",
       "...                    ...  ...               ...                   ...   \n",
       "123138                   0  ...            karl85                 14000   \n",
       "123139                  19  ...            karl85                  3750   \n",
       "123140                   0  ...            karl85                  5750   \n",
       "123141                   0  ...            karl85                  9500   \n",
       "123142                   1  ...            karl85                 32000   \n",
       "\n",
       "        correct_on_first_try                     utc_datetime    utc_date  \\\n",
       "0                      False        2020-07-15 15:27:04+00:00  2020-07-15   \n",
       "1                      False        2020-07-15 15:27:07+00:00  2020-07-15   \n",
       "2                      False        2020-07-15 15:27:15+00:00  2020-07-15   \n",
       "3                      False        2020-07-15 15:27:21+00:00  2020-07-15   \n",
       "4                      False        2020-07-15 15:27:27+00:00  2020-07-15   \n",
       "...                      ...                              ...         ...   \n",
       "123138                 False 2021-06-12 22:10:52.953572+00:00  2021-06-12   \n",
       "123139                 False 2021-06-12 22:11:00.442799+00:00  2021-06-12   \n",
       "123140                 False 2021-06-12 22:11:10.252959+00:00  2021-06-12   \n",
       "123141                 False 2021-06-12 22:11:23.885624+00:00  2021-06-12   \n",
       "123142                 False 2021-06-12 22:32:20.036637+00:00  2021-06-12   \n",
       "\n",
       "        response                                          record_id  deck_id  \\\n",
       "0           True  {\"user_id\": \"46\", \"card_id\": \"26811\", \"date\": ...        4   \n",
       "1          False  {\"user_id\": \"46\", \"card_id\": \"39163\", \"date\": ...        4   \n",
       "2          False  {\"user_id\": \"46\", \"card_id\": \"31494\", \"date\": ...        4   \n",
       "3           True  {\"user_id\": \"46\", \"card_id\": \"22848\", \"date\": ...        4   \n",
       "4           True  {\"user_id\": \"46\", \"card_id\": \"33260\", \"date\": ...        4   \n",
       "...          ...                                                ...      ...   \n",
       "123138      True  {\"user_id\": \"404\", \"card_id\": \"21394\", \"date\":...       10   \n",
       "123139      True  {\"user_id\": \"404\", \"card_id\": \"38135\", \"date\":...       11   \n",
       "123140      True  {\"user_id\": \"404\", \"card_id\": \"30418\", \"date\":...        4   \n",
       "123141     False  {\"user_id\": \"404\", \"card_id\": \"34616\", \"date\":...        4   \n",
       "123142      True  {\"user_id\": \"404\", \"card_id\": \"38453\", \"date\":...        4   \n",
       "\n",
       "             deck_name  n_minutes_spent  \n",
       "0              History                0  \n",
       "1              History                0  \n",
       "2              History                0  \n",
       "3              History                0  \n",
       "4              History                0  \n",
       "...                ...              ...  \n",
       "123138       Mythology                8  \n",
       "123139  Current Events                8  \n",
       "123140         History                8  \n",
       "123141         History                8  \n",
       "123142         History                9  \n",
       "\n",
       "[123143 rows x 35 columns]>"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_orig = df.copy()\n",
    "# df.rename(columns = {'label':'response'}, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "import uuid\n",
    "# Adapted from \n",
    "# https://stackoverflow.com/questions/66366449/how-to-generate-8-digit-unique-identifier-to-replace-the-existing-one-in-python\n",
    "original_ids = df['user_id'].unique()\n",
    "new_ids = {cid: uuid.uuid4().int for cid in original_ids}\n",
    "df['Anon Student Id'] = df['user_id'].map(new_ids)\n",
    "# Make each session id a user id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note this gets rid of milliseconds, as it seems datashop only takes up to three decimal places while %f is six\n",
    "# Only\n",
    "# df['Time'] = df['utc_date'].apply(lambda x: str(x) + \" 00:00\")\n",
    "# # df['Time'] = df['utc_date'].dt.strftime('%Y-%m-%d %H:%M:%S')\n",
    "# df['Time'].head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# No need for level? Is level different from KC\n",
    "# https://pslcdatashop.web.cmu.edu/help?page=importFormatTd\n",
    "df['Level (Deck)'] = df['deck_id'].apply(lambda x:\"Level (\" + x + \")\")\n",
    "# df['Problem Name'] = df['card_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Problem Name'] = df['card_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df['Action'] = \"Answer Question\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Condition Name'] = df['repetition_model']\n",
    "df['Time'] = df['utc_datetime']\n",
    "df['Problem Start Time'] = df['utc_datetime']\n",
    "df[\"KC (Single-KC)\"] = \"Single KC\"\n",
    "df[\"KC (Unique-Step)\"] = df['card_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Condition Type'] = \"Repetition Model\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.drop(['user_id'], inplace=True, axis=1)\n",
    "\n",
    "df[\"Condition Name 2\"] = df['deck_name']\n",
    "df[\"Condition Type 2\"] = \"Deck\"\n",
    "df[\"Outcome\"] = df['response'].apply(lambda x: \"CORRECT\" if x else \"INCORRECT\")\n",
    "# df['Step Name'] = df['usercard_n_study_total']\n",
    "df['Step Name'] = \"Flashcard\"\n",
    "# df['Session ID'] = df['index']\n",
    "# df['Session ID'] = \"1\" # Having one whole session for each student might make more sense?\n",
    "learnsphere_df = df[[\"Anon Student Id\", \"Time\", \"Level (Deck)\", \"Problem Name\", \"Step Name\", \"Outcome\", \"Condition Name\", \"Condition Type\", \"Problem Start Time\", \"KC (Single-KC)\", \"KC (Unique-Step)\", \"Condition Name 2\", \"Condition Type 2\"]]\n",
    "# TODO: Create session id\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "learnsphere_df.index.names = [\"Row\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "learnsphere_df.to_csv(\"22-08-10-learnsphere_karl.txt\", sep=\"\\t\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.4 ('scheduler-xfXz0Mu8-py3.10')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "9f7940501ef433f6039d5db707970c3cd42fba5a4613443dd730a413e02a896e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}