8_for_flowchart.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a8f83bc6-5500-40ed-89d1-3dee506b96ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "pd.set_option('max_colwidth', 100)\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from itertools import cycle\n",
    "from sklearn import svm, datasets\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import label_binarize\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import itertools\n",
    "import scipy.stats as st\n",
    "import seaborn as sns\n",
    "import scipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "75aaa97e-592f-4cf1-b7cd-15653aa4bee6",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_data = '/Users/zeleninam2/Documents/projects/cbcl_ksads_work/revision_2023/data'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ec97fc53-e81c-441e-9aa7-65092cae010d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_ksads(path_to_mydata, reporter, wave, do_print = False):\n",
    "    print('BEGIN LOADING KSADS...')\n",
    "    assert reporter in ['parent','child'], \"wrong reporter\" # reporter has to be either parent or child\n",
    "    assert wave in ['baseline', 'optimal'] # wrong wave\n",
    "    # Load baseline data\n",
    "    if reporter == 'child':\n",
    "        path_to_ksads_data = path_to_mydata+'/abcd-data-release-5.1/core/mental-health/mh_y_ksads_ss.csv'\n",
    "    elif reporter == 'parent':\n",
    "        path_to_ksads_data = path_to_mydata+'/abcd-data-release-5.1/core/mental-health/mh_p_ksads_ss.csv'        \n",
    "    ksads_data = pd.read_csv(path_to_ksads_data, sep=',', header=0, low_memory=False)\n",
    "    # Select only baseline rows\n",
    "    if wave == 'baseline':\n",
    "        ksads_data_bsl = ksads_data.loc[ksads_data.eventname == 'baseline_year_1_arm_1'].copy()\n",
    "    elif wave == 'optimal':\n",
    "        ksads_data_bsl = ksads_data.loc[ksads_data.eventname == '2_year_follow_up_y_arm_1'].copy()        \n",
    "    # check dimensions of the dataset\n",
    "    print('ORIGINAL DATA SIZE:')\n",
    "    print(ksads_data_bsl.shape)\n",
    "    # ksads - only leave columns with name id and depression diagnoses\n",
    "    if reporter == 'child':\n",
    "        depression_column_list = [f'ksads_1_{x}_t' for x in [840, 841, 843, 844, 846]]\n",
    "        adhd_column_list = ['ksads_14_856_t', 'ksads_14_855_t', 'ksads_14_853_t']\n",
    "        anxiety_column_list = [ 'ksads_5_906_t',  'ksads_5_857_t', 'ksads_6_908_t',  'ksads_7_861_t',  'ksads_7_909_t',  'ksads_8_863_t',  'ksads_8_911_t', 'ksads_10_913_t', 'ksads_10_869_t', 'ksads_9_867_t']\n",
    "    elif reporter == 'parent':\n",
    "        depression_column_list = [f'ksads_1_{x}_p' for x in [840, 841, 843, 844, 846]]\n",
    "        adhd_column_list = ['ksads_14_856_p', 'ksads_14_855_p', 'ksads_14_853_p']\n",
    "        anxiety_column_list = [ 'ksads_5_906_p',  'ksads_5_857_p', 'ksads_6_908_p',  'ksads_7_861_p',  'ksads_7_909_p',  'ksads_8_863_p',  'ksads_8_911_p', 'ksads_10_913_p', 'ksads_10_869_p', 'ksads_9_867_p']\n",
    "    col_list = ['src_subject_id']\n",
    "    col_list += depression_column_list\n",
    "    col_list += adhd_column_list\n",
    "    col_list += anxiety_column_list\n",
    "    anxiety_or_adhd_column_list = anxiety_column_list + adhd_column_list\n",
    "    ksads_bsl_id_depanxadhd = ksads_data_bsl.loc[:, col_list]\n",
    "    ksads_bsl_id_depanxadhd = ksads_bsl_id_depanxadhd.set_index('src_subject_id')\n",
    "    ksads_bsl_id_depanxadhd = ksads_bsl_id_depanxadhd.astype(float)\n",
    "    # work with missing values (could be encoded as 555 or 888)\n",
    "    # in child data, there are some columns that are fully nan. remove them\n",
    "    if reporter == 'child':\n",
    "        nan_columns = ['ksads_14_856_t', 'ksads_14_855_t', 'ksads_14_853_t', 'ksads_9_867_t', 'ksads_5_906_t', 'ksads_5_857_t',  'ksads_6_908_t', 'ksads_7_861_t', 'ksads_7_909_t']\n",
    "        # just do this once to check if everything in these columns are nans\n",
    "        #print('\\n check that nan-columns are truly nan')\n",
    "        #for c in nan_columns:\n",
    "            #print(ksads_bsl_id_depanxadhd[c].value_counts())\n",
    "        ksads_bsl_id_depanxadhd = ksads_bsl_id_depanxadhd.drop(columns = nan_columns)\n",
    "        # only keeping non-nan columns \n",
    "        adhd_column_list = []\n",
    "        if reporter == 'child':\n",
    "            anxiety_column_list = ['ksads_8_863_t', 'ksads_8_911_t', 'ksads_10_913_t', 'ksads_10_869_t']\n",
    "        col_list = []\n",
    "        col_list += depression_column_list\n",
    "        col_list += adhd_column_list\n",
    "        col_list += anxiety_column_list\n",
    "        anxiety_or_adhd_column_list = anxiety_column_list + adhd_column_list\n",
    "        ksads_bsl_id_depanxadhd = ksads_bsl_id_depanxadhd.loc[:, col_list]\n",
    "    # dealing with nan rows\n",
    "    # do this once to make sure values are 0, 1, or 555/888\n",
    "    #for c in ksads_bsl_id_depanxadhd.columns:\n",
    "        #print(ksads_bsl_id_depanxadhd[c].value_counts())\n",
    "    # if encoded as 555 or 888, replace with nan\n",
    "    ksads_bsl_id_depanxadhd[ksads_bsl_id_depanxadhd > 1.0] = np.nan\n",
    "    # [QUESTION] ???\n",
    "    #nnans = ksads_bsl_id_alldep.isnull().sum(1)\n",
    "    #print('\\nnnans')\n",
    "    #print(nnans)\n",
    "    #bad_ids = nnans[nnans > 1].index\n",
    "    #print('\\nbad ids')\n",
    "    #print(bad_ids)\n",
    "    #good_ids = nnans[nnans == 0].index\n",
    "    #ksads_dat = (ksads_bsl_id_alldep.loc[good_ids].sum(1) > 0).astype(int)\n",
    "    #print('\\ngood id sum')\n",
    "    #print(ksads_dat.sum())\n",
    "    # drop all rows that include nans\n",
    "    ksads_bsl_id_depanxadhd = ksads_bsl_id_depanxadhd.dropna()\n",
    "    if do_print:\n",
    "        print('\\nWITHOUT NANS:')\n",
    "        print(ksads_bsl_id_depanxadhd.shape)    \n",
    "    if do_print:\n",
    "        print('\\n shape after droping nans')\n",
    "        print(ksads_bsl_id_depanxadhd.shape)\n",
    "    # sum ksads scores of individual diagnoses to reach the overall ksads score, which we will use a gs\n",
    "    ksads_bsl_id_depanxadhd['any_depression'] = (ksads_bsl_id_depanxadhd.loc[:, depression_column_list] > 0).sum(1) > 0\n",
    "    ksads_bsl_id_depanxadhd['anxadhd_null'] = ksads_bsl_id_depanxadhd.loc[:, adhd_column_list + anxiety_column_list].isnull().sum(1) > 0\n",
    "    ksads_bsl_id_depanxadhd['good'] = ksads_bsl_id_depanxadhd.any_depression | (~ksads_bsl_id_depanxadhd.anxadhd_null)\n",
    "    bad_ids = ksads_bsl_id_depanxadhd.loc[~ksads_bsl_id_depanxadhd.good].index\n",
    "    good_ids = ksads_bsl_id_depanxadhd.loc[ksads_bsl_id_depanxadhd.good].index\n",
    "    if do_print:\n",
    "        check1 = (ksads_bsl_id_depanxadhd.loc[:, depression_column_list].sum(1) > 0).astype(int)\n",
    "        print('check1')\n",
    "        print(check1.sum())\n",
    "    ksads_dat = (ksads_bsl_id_depanxadhd.loc[good_ids, depression_column_list].sum(1) > 0).astype(int)\n",
    "    ksads_dat = pd.DataFrame(ksads_dat).rename(columns={0:'ksads_DEPRESSION'})\n",
    "    ksads_dat['ksads_ANXIETY'] = (ksads_bsl_id_depanxadhd.loc[good_ids, anxiety_column_list].sum(1) > 0).astype(int)\n",
    "    ksads_dat['ksads_ADHD'] = (ksads_bsl_id_depanxadhd.loc[good_ids, adhd_column_list].sum(1) > 0).astype(int)\n",
    "    ksads_dat['ksads_ADHD_OR_ANXIETY'] = (ksads_bsl_id_depanxadhd.loc[good_ids, anxiety_or_adhd_column_list].sum(1) > 0).astype(int)\n",
    "    # sometimes the sum is more than one, if more than one depression diagnosis given\n",
    "    # I want a binary value, so let's just put these to 1\n",
    "    #ksads_bsl_id_dep[ksads_bsl_id_dep > 1.0] = 1.0\n",
    "    if do_print:\n",
    "        print('\\nksads DEPRESSION value counts')\n",
    "        print(ksads_dat['ksads_DEPRESSION'].value_counts())\n",
    "    # creating different combinations of depression, anxiety, adhd comorbidities\n",
    "    DEPyes_ANXADHDno = []\n",
    "    DEPno_ANXADHDyes = []\n",
    "    for index, row in ksads_dat.iterrows():\n",
    "        if row['ksads_DEPRESSION'] == 1 and row['ksads_ADHD_OR_ANXIETY'] == 0:\n",
    "            DEPyes_ANXADHDno.append(1)\n",
    "        else:\n",
    "            DEPyes_ANXADHDno.append(0)   \n",
    "        if row['ksads_DEPRESSION'] == 0 and row['ksads_ADHD_OR_ANXIETY'] == 1:\n",
    "            DEPno_ANXADHDyes.append(1)\n",
    "        else:\n",
    "            DEPno_ANXADHDyes.append(0)\n",
    "    ksads_dat['ksads_DEPyes_ANXADHDno'] = DEPyes_ANXADHDno\n",
    "    ksads_dat['ksads_DEPno_ANXADHDyes'] = DEPno_ANXADHDyes       \n",
    "    # this is just renaming the sum depression columns\n",
    "    #ksads_bsl_id_dep = pd.DataFrame(ksads_bsl_id_dep).rename(columns = {0:'ksads_dep_binary'})\n",
    "    #if do_print:\n",
    "        #print('\\nfinal number of subjects overall')\n",
    "        #print(ksads_bsl_id_dep.shape[0])\n",
    "    if do_print:\n",
    "        print('\\nfinal sample size')\n",
    "        print(ksads_dat.shape)\n",
    "    print('\\n...FINISH LOADING KSADS')\n",
    "    return ksads_dat\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ca019587-c567-4323-b626-1661f97855c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BEGIN LOADING KSADS...\n",
      "ORIGINAL DATA SIZE:\n",
      "(11812, 1912)\n",
      "\n",
      "WITHOUT NANS:\n",
      "(11792, 9)\n",
      "\n",
      " shape after droping nans\n",
      "(11792, 9)\n",
      "check1\n",
      "119\n",
      "\n",
      "ksads DEPRESSION value counts\n",
      "ksads_DEPRESSION\n",
      "0    11673\n",
      "1      119\n",
      "Name: count, dtype: int64\n",
      "\n",
      "final sample size\n",
      "(11792, 6)\n",
      "\n",
      "...FINISH LOADING KSADS\n"
     ]
    }
   ],
   "source": [
    "data_ksads = load_ksads(path_to_data, reporter = 'child', wave = 'baseline', do_print = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96f1bcc0-18f2-4b31-b0d4-3a5da38241e4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}