Random Forest using pyspark dataframe.ipynb

{
    "nbformat_minor": 1, 
    "cells": [
        {
            "source": "\nimport ibmos2spark\n# @hidden_cell\ncredentials = {\n    'endpoint': 'https://s3-api.us-geo.objectstorage.service.networklayer.com',\n    'service_id': 'iam-ServiceId-05ab914c-2647-4655-a774-046065e40648',\n    'iam_service_endpoint': 'https://iam.bluemix.net/oidc/token',\n    'api_key': 'PGXBGSbZAGDQ0FTVUVX99kEaENnOj_hfxNqSpPo6d1MJ'\n}\n\nconfiguration_name = 'os_d5e255d53c97413bad14470512fa886c_configs'\ncos = ibmos2spark.CloudObjectStorage(sc, credentials, configuration_name, 'bluemix_cos')\n\nfrom pyspark.sql import SparkSession\nspark = SparkSession.builder.getOrCreate()\ndf = spark.read\\\n  .format('org.apache.spark.sql.execution.datasources.csv.CSVFileFormat')\\\n  .option('header', 'true')\\\n  .load(cos.url('chronic_kidney_disease_dataset.csv', 'courserads-donotdelete-pr-y8rkc0wxhkrruq'))\ndf.take(5)\n", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "Waiting for a Spark session to start...\nSpark Initialization Done! ApplicationId = app-20190301115533-0000\nKERNEL_ID = d65f9c28-b8f0-4683-b8be-5e521140ef54\n"
                }, 
                {
                    "execution_count": 1, 
                    "output_type": "execute_result", 
                    "data": {
                        "text/plain": "[Row(age='48', bp='80', sg='1.02', al='1', su='0', rbc='?', pc='normal', pcc='notpresent', ba='notpresent', bgr='121', bu='36', sc='1.2', sod='?', pot='?', hemo='15.4', pcv='44', wbcc='7800', rbcc='5.2', htn='yes', dm='yes', cad='no', appet='good', pe='no', ane='no', class='ckd', _c25=None),\n Row(age='7', bp='50', sg='1.02', al='4', su='0', rbc='?', pc='normal', pcc='notpresent', ba='notpresent', bgr='?', bu='18', sc='0.8', sod='?', pot='?', hemo='11.3', pcv='38', wbcc='6000', rbcc='?', htn='no', dm='no', cad='no', appet='good', pe='no', ane='no', class='ckd', _c25=None),\n Row(age='62', bp='80', sg='1.01', al='2', su='3', rbc='normal', pc='normal', pcc='notpresent', ba='notpresent', bgr='423', bu='53', sc='1.8', sod='?', pot='?', hemo='9.6', pcv='31', wbcc='7500', rbcc='?', htn='no', dm='yes', cad='no', appet='poor', pe='no', ane='yes', class='ckd', _c25=None),\n Row(age='48', bp='70', sg='1.005', al='4', su='0', rbc='normal', pc='abnormal', pcc='present', ba='notpresent', bgr='117', bu='56', sc='3.8', sod='111', pot='2.5', hemo='11.2', pcv='32', wbcc='6700', rbcc='3.9', htn='yes', dm='no', cad='no', appet='poor', pe='yes', ane='yes', class='ckd', _c25=None),\n Row(age='51', bp='80', sg='1.01', al='2', su='0', rbc='normal', pc='normal', pcc='notpresent', ba='notpresent', bgr='106', bu='26', sc='1.4', sod='?', pot='?', hemo='11.6', pcv='35', wbcc='7300', rbcc='4.6', htn='no', dm='no', cad='no', appet='good', pe='no', ane='no', class='ckd', _c25=None)]"
                    }, 
                    "metadata": {}
                }
            ], 
            "execution_count": 1
        }, 
        {
            "source": "# Exploratory Data Analysis", 
            "cell_type": "markdown", 
            "metadata": {}
        }, 
        {
            "source": "import numpy as np", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 2
        }, 
        {
            "source": "df.describe()", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 3, 
                    "output_type": "execute_result", 
                    "data": {
                        "text/plain": "DataFrame[summary: string, age: string, bp: string, sg: string, al: string, su: string, rbc: string, pc: string, pcc: string, ba: string, bgr: string, bu: string, sc: string, sod: string, pot: string, hemo: string, pcv: string, wbcc: string, rbcc: string, htn: string, dm: string, cad: string, appet: string, pe: string, ane: string, class: string, _c25: string]"
                    }, 
                    "metadata": {}
                }
            ], 
            "execution_count": 3
        }, 
        {
            "source": "df.show()", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "+---+---+-----+---+---+--------+--------+----------+----------+---+---+---+---+---+----+---+-----+----+---+---+---+-----+---+---+-----+----+\n|age| bp|   sg| al| su|     rbc|      pc|       pcc|        ba|bgr| bu| sc|sod|pot|hemo|pcv| wbcc|rbcc|htn| dm|cad|appet| pe|ane|class|_c25|\n+---+---+-----+---+---+--------+--------+----------+----------+---+---+---+---+---+----+---+-----+----+---+---+---+-----+---+---+-----+----+\n| 48| 80| 1.02|  1|  0|       ?|  normal|notpresent|notpresent|121| 36|1.2|  ?|  ?|15.4| 44| 7800| 5.2|yes|yes| no| good| no| no|  ckd|null|\n|  7| 50| 1.02|  4|  0|       ?|  normal|notpresent|notpresent|  ?| 18|0.8|  ?|  ?|11.3| 38| 6000|   ?| no| no| no| good| no| no|  ckd|null|\n| 62| 80| 1.01|  2|  3|  normal|  normal|notpresent|notpresent|423| 53|1.8|  ?|  ?| 9.6| 31| 7500|   ?| no|yes| no| poor| no|yes|  ckd|null|\n| 48| 70|1.005|  4|  0|  normal|abnormal|   present|notpresent|117| 56|3.8|111|2.5|11.2| 32| 6700| 3.9|yes| no| no| poor|yes|yes|  ckd|null|\n| 51| 80| 1.01|  2|  0|  normal|  normal|notpresent|notpresent|106| 26|1.4|  ?|  ?|11.6| 35| 7300| 4.6| no| no| no| good| no| no|  ckd|null|\n| 60| 90|1.015|  3|  0|       ?|       ?|notpresent|notpresent| 74| 25|1.1|142|3.2|12.2| 39| 7800| 4.4|yes|yes| no| good|yes| no|  ckd|null|\n| 68| 70| 1.01|  0|  0|       ?|  normal|notpresent|notpresent|100| 54| 24|104|  4|12.4| 36|    ?|   ?| no| no| no| good| no| no|  ckd|null|\n| 24|  ?|1.015|  2|  4|  normal|abnormal|notpresent|notpresent|410| 31|1.1|  ?|  ?|12.4| 44| 6900|   5| no|yes| no| good|yes| no|  ckd|null|\n| 52|100|1.015|  3|  0|  normal|abnormal|   present|notpresent|138| 60|1.9|  ?|  ?|10.8| 33| 9600|   4|yes|yes| no| good| no|yes|  ckd|null|\n| 53| 90| 1.02|  2|  0|abnormal|abnormal|   present|notpresent| 70|107|7.2|114|3.7| 9.5| 29|12100| 3.7|yes|yes| no| poor| no|yes|  ckd|null|\n| 50| 60| 1.01|  2|  4|       ?|abnormal|   present|notpresent|490| 55|  4|  ?|  ?| 9.4| 28|    ?|   ?|yes|yes| no| good| no|yes|  ckd|null|\n| 63| 70| 1.01|  3|  0|abnormal|abnormal|   present|notpresent|380| 60|2.7|131|4.2|10.8| 32| 4500| 3.8|yes|yes| no| poor|yes| no|  ckd|null|\n| 68| 70|1.015|  3|  1|       ?|  normal|   present|notpresent|208| 72|2.1|138|5.8| 9.7| 28|12200| 3.4|yes|yes|yes| poor|yes| no|  ckd|null|\n| 68| 70|    ?|  ?|  ?|       ?|       ?|notpresent|notpresent| 98| 86|4.6|135|3.4| 9.8|  ?|    ?|   ?|yes|yes|yes| poor|yes| no|  ckd|null|\n| 68| 80| 1.01|  3|  2|  normal|abnormal|   present|   present|157| 90|4.1|130|6.4| 5.6| 16|11000| 2.6|yes|yes|yes| poor|yes| no|  ckd|null|\n| 40| 80|1.015|  3|  0|       ?|  normal|notpresent|notpresent| 76|162|9.6|141|4.9| 7.6| 24| 3800| 2.8|yes| no| no| good| no|yes|  ckd|null|\n| 47| 70|1.015|  2|  0|       ?|  normal|notpresent|notpresent| 99| 46|2.2|138|4.1|12.6|  ?|    ?|   ?| no| no| no| good| no| no|  ckd|null|\n| 47| 80|    ?|  ?|  ?|       ?|       ?|notpresent|notpresent|114| 87|5.2|139|3.7|12.1|  ?|    ?|   ?|yes| no| no| poor| no| no|  ckd|null|\n| 60|100|1.025|  0|  3|       ?|  normal|notpresent|notpresent|263| 27|1.3|135|4.3|12.7| 37|11400| 4.3|yes|yes|yes| good| no| no|  ckd|null|\n| 62| 60|1.015|  1|  0|       ?|abnormal|   present|notpresent|100| 31|1.6|  ?|  ?|10.3| 30| 5300| 3.7|yes| no|yes| good| no| no|  ckd|null|\n+---+---+-----+---+---+--------+--------+----------+----------+---+---+---+---+---+----+---+-----+----+---+---+---+-----+---+---+-----+----+\nonly showing top 20 rows\n\n"
                }
            ], 
            "execution_count": 4
        }, 
        {
            "source": "df.columns", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 5, 
                    "output_type": "execute_result", 
                    "data": {
                        "text/plain": "['age',\n 'bp',\n 'sg',\n 'al',\n 'su',\n 'rbc',\n 'pc',\n 'pcc',\n 'ba',\n 'bgr',\n 'bu',\n 'sc',\n 'sod',\n 'pot',\n 'hemo',\n 'pcv',\n 'wbcc',\n 'rbcc',\n 'htn',\n 'dm',\n 'cad',\n 'appet',\n 'pe',\n 'ane',\n 'class',\n '_c25']"
                    }, 
                    "metadata": {}
                }
            ], 
            "execution_count": 5
        }, 
        {
            "source": "for col in df.columns:\n     df.describe([col]).show()", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "+-------+-----------------+\n|summary|              age|\n+-------+-----------------+\n|  count|              400|\n|   mean|51.48337595907928|\n| stddev|17.16971408926224|\n|    min|               11|\n|    max|                ?|\n+-------+-----------------+\n\n+-------+-----------------+\n|summary|               bp|\n+-------+-----------------+\n|  count|              400|\n|   mean|76.46907216494846|\n| stddev|13.68363749352527|\n|    min|              100|\n|    max|                ?|\n+-------+-----------------+\n\n+-------+--------------------+\n|summary|                  sg|\n+-------+--------------------+\n|  count|                 400|\n|   mean|  1.0174079320113256|\n| stddev|0.005716616974376756|\n|    min|               1.005|\n|    max|                   ?|\n+-------+--------------------+\n\n+-------+------------------+\n|summary|                al|\n+-------+------------------+\n|  count|               400|\n|   mean|1.0169491525423728|\n| stddev|1.3526789127628445|\n|    min|                 0|\n|    max|                 ?|\n+-------+------------------+\n\n+-------+-------------------+\n|summary|                 su|\n+-------+-------------------+\n|  count|                400|\n|   mean|0.45014245014245013|\n| stddev|  1.099191251885407|\n|    min|                  0|\n|    max|                  ?|\n+-------+-------------------+\n\n+-------+------+\n|summary|   rbc|\n+-------+------+\n|  count|   400|\n|   mean|  null|\n| stddev|  null|\n|    min|     ?|\n|    max|normal|\n+-------+------+\n\n+-------+------+\n|summary|    pc|\n+-------+------+\n|  count|   400|\n|   mean|  null|\n| stddev|  null|\n|    min|     ?|\n|    max|normal|\n+-------+------+\n\n+-------+-------+\n|summary|    pcc|\n+-------+-------+\n|  count|    400|\n|   mean|   null|\n| stddev|   null|\n|    min|      ?|\n|    max|present|\n+-------+-------+\n\n+-------+-------+\n|summary|     ba|\n+-------+-------+\n|  count|    400|\n|   mean|   null|\n| stddev|   null|\n|    min|      ?|\n|    max|present|\n+-------+-------+\n\n+-------+-----------------+\n|summary|              bgr|\n+-------+-----------------+\n|  count|              400|\n|   mean|148.0365168539326|\n| stddev|79.28171423511773|\n|    min|              100|\n|    max|                ?|\n+-------+-----------------+\n\n+-------+------------------+\n|summary|                bu|\n+-------+------------------+\n|  count|               400|\n|   mean|57.425721784776904|\n| stddev|50.503005849222504|\n|    min|               1.5|\n|    max|                 ?|\n+-------+------------------+\n\n+-------+------------------+\n|summary|                sc|\n+-------+------------------+\n|  count|               400|\n|   mean|3.0724543080939934|\n| stddev| 5.741126066859789|\n|    min|               0.4|\n|    max|                 ?|\n+-------+------------------+\n\n+-------+------------------+\n|summary|               sod|\n+-------+------------------+\n|  count|               400|\n|   mean|137.52875399361022|\n| stddev|10.408752051798777|\n|    min|               104|\n|    max|                 ?|\n+-------+------------------+\n\n+-------+-----------------+\n|summary|              pot|\n+-------+-----------------+\n|  count|              400|\n|   mean|4.627243589743592|\n| stddev|3.193904176556695|\n|    min|              2.5|\n|    max|                ?|\n+-------+-----------------+\n\n+-------+------------------+\n|summary|              hemo|\n+-------+------------------+\n|  count|               400|\n|   mean|12.526436781609195|\n| stddev| 2.912586608826765|\n|    min|                10|\n|    max|                 ?|\n+-------+------------------+\n\n+-------+-----------------+\n|summary|              pcv|\n+-------+-----------------+\n|  count|              400|\n|   mean|38.88449848024316|\n| stddev|8.990104814740933|\n|    min|              \t43|\n|    max|                ?|\n+-------+-----------------+\n\n+-------+------------------+\n|summary|              wbcc|\n+-------+------------------+\n|  count|               400|\n|   mean| 8406.122448979591|\n| stddev|2944.4741904103385|\n|    min|             \t6200|\n|    max|                 ?|\n+-------+------------------+\n\n+-------+------------------+\n|summary|              rbcc|\n+-------+------------------+\n|  count|               400|\n|   mean| 4.707434944237919|\n| stddev|1.0253232655721791|\n|    min|                \t?|\n|    max|                 ?|\n+-------+------------------+\n\n+-------+----+\n|summary| htn|\n+-------+----+\n|  count| 400|\n|   mean|null|\n| stddev|null|\n|    min|   ?|\n|    max| yes|\n+-------+----+\n\n+-------+----+\n|summary|  dm|\n+-------+----+\n|  count| 399|\n|   mean|null|\n| stddev|null|\n|    min| \tno|\n|    max| yes|\n+-------+----+\n\n+-------+----+\n|summary| cad|\n+-------+----+\n|  count| 400|\n|   mean|null|\n| stddev|null|\n|    min| \tno|\n|    max| yes|\n+-------+----+\n\n+-------+-----+\n|summary|appet|\n+-------+-----+\n|  count|  400|\n|   mean| null|\n| stddev| null|\n|    min|    ?|\n|    max| poor|\n+-------+-----+\n\n+-------+----+\n|summary|  pe|\n+-------+----+\n|  count| 400|\n|   mean|null|\n| stddev|null|\n|    min|   ?|\n|    max| yes|\n+-------+----+\n\n+-------+----+\n|summary| ane|\n+-------+----+\n|  count| 400|\n|   mean|null|\n| stddev|null|\n|    min|   ?|\n|    max| yes|\n+-------+----+\n\n+-------+------+\n|summary| class|\n+-------+------+\n|  count|   400|\n|   mean|  null|\n| stddev|  null|\n|    min|   ckd|\n|    max|notckd|\n+-------+------+\n\n+-------+------+\n|summary|  _c25|\n+-------+------+\n|  count|     1|\n|   mean|  null|\n| stddev|  null|\n|    min|notckd|\n|    max|notckd|\n+-------+------+\n\n"
                }
            ], 
            "execution_count": 6
        }, 
        {
            "source": "df = df.drop(\"_c25\")", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 7
        }, 
        {
            "source": "for col in df.columns:\n     df.describe([col]).show()", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "+-------+-----------------+\n|summary|              age|\n+-------+-----------------+\n|  count|              400|\n|   mean|51.48337595907928|\n| stddev|17.16971408926224|\n|    min|               11|\n|    max|                ?|\n+-------+-----------------+\n\n+-------+-----------------+\n|summary|               bp|\n+-------+-----------------+\n|  count|              400|\n|   mean|76.46907216494846|\n| stddev|13.68363749352527|\n|    min|              100|\n|    max|                ?|\n+-------+-----------------+\n\n+-------+--------------------+\n|summary|                  sg|\n+-------+--------------------+\n|  count|                 400|\n|   mean|  1.0174079320113256|\n| stddev|0.005716616974376756|\n|    min|               1.005|\n|    max|                   ?|\n+-------+--------------------+\n\n+-------+------------------+\n|summary|                al|\n+-------+------------------+\n|  count|               400|\n|   mean|1.0169491525423728|\n| stddev|1.3526789127628445|\n|    min|                 0|\n|    max|                 ?|\n+-------+------------------+\n\n+-------+-------------------+\n|summary|                 su|\n+-------+-------------------+\n|  count|                400|\n|   mean|0.45014245014245013|\n| stddev|  1.099191251885407|\n|    min|                  0|\n|    max|                  ?|\n+-------+-------------------+\n\n+-------+------+\n|summary|   rbc|\n+-------+------+\n|  count|   400|\n|   mean|  null|\n| stddev|  null|\n|    min|     ?|\n|    max|normal|\n+-------+------+\n\n+-------+------+\n|summary|    pc|\n+-------+------+\n|  count|   400|\n|   mean|  null|\n| stddev|  null|\n|    min|     ?|\n|    max|normal|\n+-------+------+\n\n+-------+-------+\n|summary|    pcc|\n+-------+-------+\n|  count|    400|\n|   mean|   null|\n| stddev|   null|\n|    min|      ?|\n|    max|present|\n+-------+-------+\n\n+-------+-------+\n|summary|     ba|\n+-------+-------+\n|  count|    400|\n|   mean|   null|\n| stddev|   null|\n|    min|      ?|\n|    max|present|\n+-------+-------+\n\n+-------+-----------------+\n|summary|              bgr|\n+-------+-----------------+\n|  count|              400|\n|   mean|148.0365168539326|\n| stddev|79.28171423511773|\n|    min|              100|\n|    max|                ?|\n+-------+-----------------+\n\n+-------+------------------+\n|summary|                bu|\n+-------+------------------+\n|  count|               400|\n|   mean|57.425721784776904|\n| stddev|50.503005849222504|\n|    min|               1.5|\n|    max|                 ?|\n+-------+------------------+\n\n+-------+------------------+\n|summary|                sc|\n+-------+------------------+\n|  count|               400|\n|   mean|3.0724543080939934|\n| stddev| 5.741126066859789|\n|    min|               0.4|\n|    max|                 ?|\n+-------+------------------+\n\n+-------+------------------+\n|summary|               sod|\n+-------+------------------+\n|  count|               400|\n|   mean|137.52875399361022|\n| stddev|10.408752051798777|\n|    min|               104|\n|    max|                 ?|\n+-------+------------------+\n\n+-------+-----------------+\n|summary|              pot|\n+-------+-----------------+\n|  count|              400|\n|   mean|4.627243589743592|\n| stddev|3.193904176556695|\n|    min|              2.5|\n|    max|                ?|\n+-------+-----------------+\n\n+-------+------------------+\n|summary|              hemo|\n+-------+------------------+\n|  count|               400|\n|   mean|12.526436781609195|\n| stddev| 2.912586608826765|\n|    min|                10|\n|    max|                 ?|\n+-------+------------------+\n\n+-------+-----------------+\n|summary|              pcv|\n+-------+-----------------+\n|  count|              400|\n|   mean|38.88449848024316|\n| stddev|8.990104814740933|\n|    min|              \t43|\n|    max|                ?|\n+-------+-----------------+\n\n+-------+------------------+\n|summary|              wbcc|\n+-------+------------------+\n|  count|               400|\n|   mean| 8406.122448979591|\n| stddev|2944.4741904103385|\n|    min|             \t6200|\n|    max|                 ?|\n+-------+------------------+\n\n+-------+------------------+\n|summary|              rbcc|\n+-------+------------------+\n|  count|               400|\n|   mean| 4.707434944237919|\n| stddev|1.0253232655721791|\n|    min|                \t?|\n|    max|                 ?|\n+-------+------------------+\n\n+-------+----+\n|summary| htn|\n+-------+----+\n|  count| 400|\n|   mean|null|\n| stddev|null|\n|    min|   ?|\n|    max| yes|\n+-------+----+\n\n+-------+----+\n|summary|  dm|\n+-------+----+\n|  count| 399|\n|   mean|null|\n| stddev|null|\n|    min| \tno|\n|    max| yes|\n+-------+----+\n\n+-------+----+\n|summary| cad|\n+-------+----+\n|  count| 400|\n|   mean|null|\n| stddev|null|\n|    min| \tno|\n|    max| yes|\n+-------+----+\n\n+-------+-----+\n|summary|appet|\n+-------+-----+\n|  count|  400|\n|   mean| null|\n| stddev| null|\n|    min|    ?|\n|    max| poor|\n+-------+-----+\n\n+-------+----+\n|summary|  pe|\n+-------+----+\n|  count| 400|\n|   mean|null|\n| stddev|null|\n|    min|   ?|\n|    max| yes|\n+-------+----+\n\n+-------+----+\n|summary| ane|\n+-------+----+\n|  count| 400|\n|   mean|null|\n| stddev|null|\n|    min|   ?|\n|    max| yes|\n+-------+----+\n\n+-------+------+\n|summary| class|\n+-------+------+\n|  count|   400|\n|   mean|  null|\n| stddev|  null|\n|    min|   ckd|\n|    max|notckd|\n+-------+------+\n\n"
                }
            ], 
            "execution_count": 8
        }, 
        {
            "source": "#check unique values for class\ndf.select('class').distinct().rdd.map(lambda r: r[0]).collect()", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 9, 
                    "output_type": "execute_result", 
                    "data": {
                        "text/plain": "['notckd', 'ckd', 'no', 'ckd\\t']"
                    }, 
                    "metadata": {}
                }
            ], 
            "execution_count": 9
        }, 
        {
            "source": "#filter out those rows\ndf.filter(df['class']==\"ckd\\t\").show()", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "+---+---+----+---+---+------+--------+----------+----------+---+---+---+---+---+----+---+----+----+---+---+---+-----+---+---+-----+\n|age| bp|  sg| al| su|   rbc|      pc|       pcc|        ba|bgr| bu| sc|sod|pot|hemo|pcv|wbcc|rbcc|htn| dm|cad|appet| pe|ane|class|\n+---+---+----+---+---+------+--------+----------+----------+---+---+---+---+---+----+---+----+----+---+---+---+-----+---+---+-----+\n| 72| 80|   ?|  ?|  ?|     ?|       ?|notpresent|notpresent|137| 65|3.4|141|4.7| 9.7| 28|6900| 2.5|yes|yes| no| poor| no|yes| ckd\t|\n| 65| 60|1.01|  2|  0|normal|abnormal|   present|notpresent|192| 17|1.7|130|4.3|   ?|  ?|9500|   ?|yes|yes| no| poor| no| no| ckd\t|\n+---+---+----+---+---+------+--------+----------+----------+---+---+---+---+---+----+---+----+----+---+---+---+-----+---+---+-----+\n\n"
                }
            ], 
            "execution_count": 10
        }, 
        {
            "source": "#since 'ckd\\t' is the same class as of 'ckd', so we replaced first to correct one.\nfrom pyspark.sql.functions import when, lit\ndf = df.withColumn('class', when(df['class']=='ckd\\t', \nlit(\"ckd\")).otherwise(df['class']))\n", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 11
        }, 
        {
            "source": "#Validate that there is no more left after replacing\ndf.filter(df['class']==\"ckd\\t\").show()", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "+---+---+---+---+---+---+---+---+---+---+---+---+---+---+----+---+----+----+---+---+---+-----+---+---+-----+\n|age| bp| sg| al| su|rbc| pc|pcc| ba|bgr| bu| sc|sod|pot|hemo|pcv|wbcc|rbcc|htn| dm|cad|appet| pe|ane|class|\n+---+---+---+---+---+---+---+---+---+---+---+---+---+---+----+---+----+----+---+---+---+-----+---+---+-----+\n+---+---+---+---+---+---+---+---+---+---+---+---+---+---+----+---+----+----+---+---+---+-----+---+---+-----+\n\n"
                }
            ], 
            "execution_count": 12
        }, 
        {
            "source": "#replace no with notckd of class column\nfrom pyspark.sql.functions import when, lit\ndf = df.withColumn('class', when(df['class']=='no', \nlit(\"notckd\")).otherwise(df['class']))\n", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 13
        }, 
        {
            "source": "#validate now  unique values for class\ndf.select('class').distinct().rdd.map(lambda r: r[0]).collect()", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 14, 
                    "output_type": "execute_result", 
                    "data": {
                        "text/plain": "['notckd', 'ckd']"
                    }, 
                    "metadata": {}
                }
            ], 
            "execution_count": 14
        }, 
        {
            "source": "#Count the value of null in every column\nfor col in df.columns:\n    print(col, \"\\t\", \"with null values: \", df.filter(df[col].isNull()).count())\n    \n", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "age \t with null values:  0\nbp \t with null values:  0\nsg \t with null values:  0\nal \t with null values:  0\nsu \t with null values:  0\nrbc \t with null values:  0\npc \t with null values:  0\npcc \t with null values:  0\nba \t with null values:  0\nbgr \t with null values:  0\nbu \t with null values:  0\nsc \t with null values:  0\nsod \t with null values:  0\npot \t with null values:  0\nhemo \t with null values:  0\npcv \t with null values:  0\nwbcc \t with null values:  0\nrbcc \t with null values:  0\nhtn \t with null values:  0\ndm \t with null values:  1\ncad \t with null values:  0\nappet \t with null values:  0\npe \t with null values:  0\nane \t with null values:  0\nclass \t with null values:  0\n"
                }
            ], 
            "execution_count": 15
        }, 
        {
            "source": "#Count the value of '?' in every column\nfor col in df.columns:\n    print(col, \"\\t\", \"with '?' values: \", df.filter(df[col]==\"?\").count())", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "age \t with '?' values:  9\nbp \t with '?' values:  12\nsg \t with '?' values:  47\nal \t with '?' values:  46\nsu \t with '?' values:  49\nrbc \t with '?' values:  152\npc \t with '?' values:  65\npcc \t with '?' values:  4\nba \t with '?' values:  4\nbgr \t with '?' values:  44\nbu \t with '?' values:  19\nsc \t with '?' values:  17\nsod \t with '?' values:  87\npot \t with '?' values:  88\nhemo \t with '?' values:  52\npcv \t with '?' values:  70\nwbcc \t with '?' values:  105\nrbcc \t with '?' values:  130\nhtn \t with '?' values:  2\ndm \t with '?' values:  2\ncad \t with '?' values:  2\nappet \t with '?' values:  1\npe \t with '?' values:  1\nane \t with '?' values:  1\nclass \t with '?' values:  0\n"
                }
            ], 
            "execution_count": 16
        }, 
        {
            "source": "#since in our dataset, column value is represented through '?' and None, so we will first replace the '? with None values.'\ndf = df.replace('?',None)\ndf = df.replace('\\t?',None)\ndf = df.replace('?\\t',None)\ndf = df.replace('\\tno',\"no\")\ndf = df.replace('no\\t',\"no\")\ndf = df.replace('\\tyes',\"yes\")\ndf = df.replace('yes\\t',\"yes\")", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 17
        }, 
        {
            "source": "#Now for validation Count the value of '?' in every column\nfor col in df.columns:\n    print(col, \"\\t\", \"with '?' values: \", df.filter(df[col]==\"?\").count())", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "age \t with '?' values:  0\nbp \t with '?' values:  0\nsg \t with '?' values:  0\nal \t with '?' values:  0\nsu \t with '?' values:  0\nrbc \t with '?' values:  0\npc \t with '?' values:  0\npcc \t with '?' values:  0\nba \t with '?' values:  0\nbgr \t with '?' values:  0\nbu \t with '?' values:  0\nsc \t with '?' values:  0\nsod \t with '?' values:  0\npot \t with '?' values:  0\nhemo \t with '?' values:  0\npcv \t with '?' values:  0\nwbcc \t with '?' values:  0\nrbcc \t with '?' values:  0\nhtn \t with '?' values:  0\ndm \t with '?' values:  0\ncad \t with '?' values:  0\nappet \t with '?' values:  0\npe \t with '?' values:  0\nane \t with '?' values:  0\nclass \t with '?' values:  0\n"
                }
            ], 
            "execution_count": 18
        }, 
        {
            "source": "#Now for validation Count the value of null in every column\nfor col in df.columns:\n    print(col, \"\\t\", \"with null values: \", df.filter(df[col].isNull()).count())\n    ", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "age \t with null values:  9\nbp \t with null values:  12\nsg \t with null values:  47\nal \t with null values:  46\nsu \t with null values:  49\nrbc \t with null values:  152\npc \t with null values:  65\npcc \t with null values:  4\nba \t with null values:  4\nbgr \t with null values:  44\nbu \t with null values:  19\nsc \t with null values:  17\nsod \t with null values:  87\npot \t with null values:  88\nhemo \t with null values:  52\npcv \t with null values:  71\nwbcc \t with null values:  106\nrbcc \t with null values:  131\nhtn \t with null values:  2\ndm \t with null values:  3\ncad \t with null values:  2\nappet \t with null values:  1\npe \t with null values:  1\nane \t with null values:  1\nclass \t with null values:  0\n"
                }
            ], 
            "execution_count": 19
        }, 
        {
            "source": "#Computation for mean", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 20
        }, 
        {
            "source": "#Find the average of a column\n\nfrom pyspark.sql.functions import avg\n\nmean_value = df.select(avg(df['age']))", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 21
        }, 
        {
            "source": "mean_value", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 22, 
                    "output_type": "execute_result", 
                    "data": {
                        "text/plain": "DataFrame[avg(age): double]"
                    }, 
                    "metadata": {}
                }
            ], 
            "execution_count": 22
        }, 
        {
            "source": "mean_value.columns", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 23, 
                    "output_type": "execute_result", 
                    "data": {
                        "text/plain": "['avg(age)']"
                    }, 
                    "metadata": {}
                }
            ], 
            "execution_count": 23
        }, 
        {
            "source": "res = mean_value.rdd.map(lambda row : row['avg(age)']).collect()\n\n", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 24
        }, 
        {
            "source": "res", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 25, 
                    "output_type": "execute_result", 
                    "data": {
                        "text/plain": "[51.48337595907928]"
                    }, 
                    "metadata": {}
                }
            ], 
            "execution_count": 25
        }, 
        {
            "source": "#Complete function to find the mean of columns of pyspark dataframe", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 26
        }, 
        {
            "source": "#Find the avg of all numeric columns\nfrom pyspark.sql.functions import avg\n\ndef mean_of_pyspark_columns(df, numeric_cols, verbose=False):\n    col_with_mean=[]\n    for col in numeric_cols:\n        mean_value = df.select(avg(df[col]))\n        avg_col = mean_value.columns[0]\n        res = mean_value.rdd.map(lambda row : row[avg_col]).collect()\n        \n        if (verbose==True): print(mean_value.columns[0], \"\\t\", res[0])\n        col_with_mean.append([col, res[0]])    \n    return col_with_mean   \n    \n", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 27
        }, 
        {
            "source": "#Validate it\nnumeric_cols=['age', 'bp', 'sg','bgr', 'bu', 'sc', 'sod', 'pot', 'hemo', 'pcv', 'wbcc', 'rbcc']    \nmean_of_pyspark_columns(df, numeric_cols, verbose=True)    ", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "avg(age) \t 51.48337595907928\navg(bp) \t 76.46907216494846\navg(sg) \t 1.0174079320113256\navg(bgr) \t 148.0365168539326\navg(bu) \t 57.425721784776904\navg(sc) \t 3.0724543080939934\navg(sod) \t 137.52875399361022\navg(pot) \t 4.627243589743592\navg(hemo) \t 12.526436781609195\navg(pcv) \t 38.88449848024316\navg(wbcc) \t 8406.122448979591\navg(rbcc) \t 4.707434944237919\n"
                }, 
                {
                    "execution_count": 28, 
                    "output_type": "execute_result", 
                    "data": {
                        "text/plain": "[['age', 51.48337595907928],\n ['bp', 76.46907216494846],\n ['sg', 1.0174079320113256],\n ['bgr', 148.0365168539326],\n ['bu', 57.425721784776904],\n ['sc', 3.0724543080939934],\n ['sod', 137.52875399361022],\n ['pot', 4.627243589743592],\n ['hemo', 12.526436781609195],\n ['pcv', 38.88449848024316],\n ['wbcc', 8406.122448979591],\n ['rbcc', 4.707434944237919]]"
                    }, 
                    "metadata": {}
                }
            ], 
            "execution_count": 28
        }, 
        {
            "source": "#Complete function to find the mode of columns of pyspark dataframe", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 29
        }, 
        {
            "source": "def mode_of_pyspark_columns(df, cat_col_list, verbose=False):\n    col_with_mode=[]\n    for col in cat_col_list:\n        #Filter null\n        df = df.filter(df[col].isNull()==False)\n        #Find unique_values_with_count\n        unique_classes = df.select(col).distinct().rdd.map(lambda x: x[0]).collect()\n        unique_values_with_count=[]\n        for uc in unique_classes:\n             unique_values_with_count.append([uc, df.filter(df[col]==uc).count()])\n        #sort unique values w.r.t their count values\n        sorted_unique_values_with_count= sorted(unique_values_with_count, key = lambda x: x[1], reverse =True)\n        \n        if (verbose==True): print(col, sorted_unique_values_with_count, \" and mode is \", sorted_unique_values_with_count[0][0])\n        col_with_mode.append([col, sorted_unique_values_with_count[0][0]])\n    return col_with_mode", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 30
        }, 
        {
            "source": "#Validate it\ncat_col_list=['al', 'su', 'rbc', 'pc', 'pcc', 'ba', 'htn', 'dm', 'cad', 'appet', 'pe', 'ane']\nmode_of_pyspark_columns(df, cat_col_list, verbose=True)", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "al [['0', 199], ['1', 44], ['3', 43], ['2', 43], ['4', 24], ['5', 1]]  and mode is  0\nsu [['0', 290], ['2', 18], ['3', 14], ['1', 13], ['4', 13], ['5', 3]]  and mode is  0\nrbc [['normal', 198], ['abnormal', 47]]  and mode is  normal\npc [['normal', 180], ['abnormal', 58]]  and mode is  normal\npcc [['notpresent', 207], ['present', 28]]  and mode is  notpresent\nba [['notpresent', 218], ['present', 17]]  and mode is  notpresent\nhtn [['no', 172], ['yes', 61]]  and mode is  no\ndm [['no', 181], ['yes', 51]]  and mode is  no\ncad [['no', 216], ['yes', 16]]  and mode is  no\nappet [['good', 191], ['poor', 40]]  and mode is  good\npe [['no', 190], ['yes', 41]]  and mode is  no\nane [['no', 201], ['yes', 30]]  and mode is  no\n"
                }, 
                {
                    "execution_count": 31, 
                    "output_type": "execute_result", 
                    "data": {
                        "text/plain": "[['al', '0'],\n ['su', '0'],\n ['rbc', 'normal'],\n ['pc', 'normal'],\n ['pcc', 'notpresent'],\n ['ba', 'notpresent'],\n ['htn', 'no'],\n ['dm', 'no'],\n ['cad', 'no'],\n ['appet', 'good'],\n ['pe', 'no'],\n ['ane', 'no']]"
                    }, 
                    "metadata": {}
                }
            ], 
            "execution_count": 31
        }, 
        {
            "source": "#as you can see from the above that we have to replace for colmun 'cad' also\nfrom pyspark.sql.functions import when, lit\ndf = df.withColumn('cad', when(df['cad']=='\\tno', \nlit(\"no\")).otherwise(df['cad']))", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 32
        }, 
        {
            "source": "df.select('cad').distinct().show()", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "+----+\n| cad|\n+----+\n|null|\n|  no|\n| yes|\n+----+\n\n"
                }
            ], 
            "execution_count": 33
        }, 
        {
            "source": "#Now we have found our bits and pieces, its time to integrate them\n\n#Fill missing values for mean and mode", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 34
        }, 
        {
            "source": "#Fill missing values for mean\nfrom pyspark.sql.functions import when, lit\n\ndef fill_missing_with_mean(df, numeric_cols):\n    col_with_mean = mean_of_pyspark_columns(df, numeric_cols) \n    \n    for col, mean in col_with_mean:\n        df = df.withColumn(col, when(df[col].isNull()==True, \n        lit(mean)).otherwise(df[col]))\n        \n    return df", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 35
        }, 
        {
            "source": "#Now this is the time to literally consume the fill_missing_with_mean\n\nnumeric_cols=['age', 'bp', 'sg','bgr', 'bu', 'sc', 'sod', 'pot', 'hemo', 'pcv', 'wbcc', 'rbcc']     \ndf = fill_missing_with_mean(df, numeric_cols)", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 36
        }, 
        {
            "source": "df.show()", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "+---+-----------------+------------------+----+----+--------+--------+----------+----------+-----------------+---+---+------------------+-----------------+----+-----------------+-----------------+-----------------+---+---+---+-----+---+---+-----+\n|age|               bp|                sg|  al|  su|     rbc|      pc|       pcc|        ba|              bgr| bu| sc|               sod|              pot|hemo|              pcv|             wbcc|             rbcc|htn| dm|cad|appet| pe|ane|class|\n+---+-----------------+------------------+----+----+--------+--------+----------+----------+-----------------+---+---+------------------+-----------------+----+-----------------+-----------------+-----------------+---+---+---+-----+---+---+-----+\n| 48|               80|              1.02|   1|   0|    null|  normal|notpresent|notpresent|              121| 36|1.2|137.52875399361022|4.627243589743592|15.4|               44|             7800|              5.2|yes|yes| no| good| no| no|  ckd|\n|  7|               50|              1.02|   4|   0|    null|  normal|notpresent|notpresent|148.0365168539326| 18|0.8|137.52875399361022|4.627243589743592|11.3|               38|             6000|4.707434944237919| no| no| no| good| no| no|  ckd|\n| 62|               80|              1.01|   2|   3|  normal|  normal|notpresent|notpresent|              423| 53|1.8|137.52875399361022|4.627243589743592| 9.6|               31|             7500|4.707434944237919| no|yes| no| poor| no|yes|  ckd|\n| 48|               70|             1.005|   4|   0|  normal|abnormal|   present|notpresent|              117| 56|3.8|               111|              2.5|11.2|               32|             6700|              3.9|yes| no| no| poor|yes|yes|  ckd|\n| 51|               80|              1.01|   2|   0|  normal|  normal|notpresent|notpresent|              106| 26|1.4|137.52875399361022|4.627243589743592|11.6|               35|             7300|              4.6| no| no| no| good| no| no|  ckd|\n| 60|               90|             1.015|   3|   0|    null|    null|notpresent|notpresent|               74| 25|1.1|               142|              3.2|12.2|               39|             7800|              4.4|yes|yes| no| good|yes| no|  ckd|\n| 68|               70|              1.01|   0|   0|    null|  normal|notpresent|notpresent|              100| 54| 24|               104|                4|12.4|               36|8406.122448979591|4.707434944237919| no| no| no| good| no| no|  ckd|\n| 24|76.46907216494846|             1.015|   2|   4|  normal|abnormal|notpresent|notpresent|              410| 31|1.1|137.52875399361022|4.627243589743592|12.4|               44|             6900|                5| no|yes| no| good|yes| no|  ckd|\n| 52|              100|             1.015|   3|   0|  normal|abnormal|   present|notpresent|              138| 60|1.9|137.52875399361022|4.627243589743592|10.8|               33|             9600|                4|yes|yes| no| good| no|yes|  ckd|\n| 53|               90|              1.02|   2|   0|abnormal|abnormal|   present|notpresent|               70|107|7.2|               114|              3.7| 9.5|               29|            12100|              3.7|yes|yes| no| poor| no|yes|  ckd|\n| 50|               60|              1.01|   2|   4|    null|abnormal|   present|notpresent|              490| 55|  4|137.52875399361022|4.627243589743592| 9.4|               28|8406.122448979591|4.707434944237919|yes|yes| no| good| no|yes|  ckd|\n| 63|               70|              1.01|   3|   0|abnormal|abnormal|   present|notpresent|              380| 60|2.7|               131|              4.2|10.8|               32|             4500|              3.8|yes|yes| no| poor|yes| no|  ckd|\n| 68|               70|             1.015|   3|   1|    null|  normal|   present|notpresent|              208| 72|2.1|               138|              5.8| 9.7|               28|            12200|              3.4|yes|yes|yes| poor|yes| no|  ckd|\n| 68|               70|1.0174079320113256|null|null|    null|    null|notpresent|notpresent|               98| 86|4.6|               135|              3.4| 9.8|38.88449848024316|8406.122448979591|4.707434944237919|yes|yes|yes| poor|yes| no|  ckd|\n| 68|               80|              1.01|   3|   2|  normal|abnormal|   present|   present|              157| 90|4.1|               130|              6.4| 5.6|               16|            11000|              2.6|yes|yes|yes| poor|yes| no|  ckd|\n| 40|               80|             1.015|   3|   0|    null|  normal|notpresent|notpresent|               76|162|9.6|               141|              4.9| 7.6|               24|             3800|              2.8|yes| no| no| good| no|yes|  ckd|\n| 47|               70|             1.015|   2|   0|    null|  normal|notpresent|notpresent|               99| 46|2.2|               138|              4.1|12.6|38.88449848024316|8406.122448979591|4.707434944237919| no| no| no| good| no| no|  ckd|\n| 47|               80|1.0174079320113256|null|null|    null|    null|notpresent|notpresent|              114| 87|5.2|               139|              3.7|12.1|38.88449848024316|8406.122448979591|4.707434944237919|yes| no| no| poor| no| no|  ckd|\n| 60|              100|             1.025|   0|   3|    null|  normal|notpresent|notpresent|              263| 27|1.3|               135|              4.3|12.7|               37|            11400|              4.3|yes|yes|yes| good| no| no|  ckd|\n| 62|               60|             1.015|   1|   0|    null|abnormal|   present|notpresent|              100| 31|1.6|137.52875399361022|4.627243589743592|10.3|               30|             5300|              3.7|yes| no|yes| good| no| no|  ckd|\n+---+-----------------+------------------+----+----+--------+--------+----------+----------+-----------------+---+---+------------------+-----------------+----+-----------------+-----------------+-----------------+---+---+---+-----+---+---+-----+\nonly showing top 20 rows\n\n"
                }
            ], 
            "execution_count": 37
        }, 
        {
            "source": "#Fill missing values for mode\nfrom pyspark.sql.functions import when, lit\n\ndef fill_missing_with_mode(df, cat_col_list):\n    col_with_mode =mode_of_pyspark_columns(df, cat_col_list)\n    \n    for col, mode in col_with_mode:\n        df = df.withColumn(col, when(df[col].isNull()==True, \n        lit(mode)).otherwise(df[col]))\n        \n    return df", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 38
        }, 
        {
            "source": "#Now this is the time to literally consume the fill_missing_with_mode\ncat_col_list=['al', 'su', 'rbc', 'pc', 'pcc', 'ba', 'htn', 'dm', 'cad', 'appet', 'pe', 'ane']\ndf = fill_missing_with_mode(df, cat_col_list)", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 39
        }, 
        {
            "source": "#again validate that our awesome functions worked or not on true dataframe that is 'df'\nfor col in df.columns:\n     df.describe([col]).show()", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "+-------+----------------+\n|summary|             age|\n+-------+----------------+\n|  count|             400|\n|   mean|51.4833759590793|\n| stddev|16.9749662313574|\n|    min|              11|\n|    max|              90|\n+-------+----------------+\n\n+-------+-----------------+\n|summary|               bp|\n+-------+-----------------+\n|  count|              400|\n|   mean|76.46907216494844|\n| stddev|13.47629766150936|\n|    min|              100|\n|    max|               90|\n+-------+-----------------+\n\n+-------+--------------------+\n|summary|                  sg|\n+-------+--------------------+\n|  count|                 400|\n|   mean|   1.017407932011323|\n| stddev|0.005369377992466434|\n|    min|               1.005|\n|    max|               1.025|\n+-------+--------------------+\n\n+-------+------------------+\n|summary|                al|\n+-------+------------------+\n|  count|               400|\n|   mean|               0.9|\n| stddev|1.3131301447085713|\n|    min|                 0|\n|    max|                 5|\n+-------+------------------+\n\n+-------+-----------------+\n|summary|               su|\n+-------+-----------------+\n|  count|              400|\n|   mean|            0.395|\n| stddev|1.040038075262145|\n|    min|                0|\n|    max|                5|\n+-------+-----------------+\n\n+-------+--------+\n|summary|     rbc|\n+-------+--------+\n|  count|     400|\n|   mean|    null|\n| stddev|    null|\n|    min|abnormal|\n|    max|  normal|\n+-------+--------+\n\n+-------+--------+\n|summary|      pc|\n+-------+--------+\n|  count|     400|\n|   mean|    null|\n| stddev|    null|\n|    min|abnormal|\n|    max|  normal|\n+-------+--------+\n\n+-------+----------+\n|summary|       pcc|\n+-------+----------+\n|  count|       400|\n|   mean|      null|\n| stddev|      null|\n|    min|notpresent|\n|    max|   present|\n+-------+----------+\n\n+-------+----------+\n|summary|        ba|\n+-------+----------+\n|  count|       400|\n|   mean|      null|\n| stddev|      null|\n|    min|notpresent|\n|    max|   present|\n+-------+----------+\n\n+-------+------------------+\n|summary|               bgr|\n+-------+------------------+\n|  count|               400|\n|   mean|148.03651685393265|\n| stddev| 74.78263447653052|\n|    min|               100|\n|    max|                99|\n+-------+------------------+\n\n+-------+------------------+\n|summary|                bu|\n+-------+------------------+\n|  count|               400|\n|   mean| 57.42572178477692|\n| stddev|49.285887092376456|\n|    min|               1.5|\n|    max|              98.6|\n+-------+------------------+\n\n+-------+-----------------+\n|summary|               sc|\n+-------+-----------------+\n|  count|              400|\n|   mean|3.072454308093993|\n| stddev|5.617490117519301|\n|    min|              0.4|\n|    max|              9.7|\n+-------+-----------------+\n\n+-------+-----------------+\n|summary|              sod|\n+-------+-----------------+\n|  count|              400|\n|   mean|137.5287539936103|\n| stddev| 9.20427344548673|\n|    min|              104|\n|    max|              4.5|\n+-------+-----------------+\n\n+-------+------------------+\n|summary|               pot|\n+-------+------------------+\n|  count|               400|\n|   mean| 4.627243589743589|\n| stddev|2.8197826172844933|\n|    min|               2.5|\n|    max|               7.6|\n+-------+------------------+\n\n+-------+------------------+\n|summary|              hemo|\n+-------+------------------+\n|  count|               400|\n|   mean|12.526436781609211|\n| stddev|2.7161711873186567|\n|    min|                10|\n|    max|               9.9|\n+-------+------------------+\n\n+-------+------------------+\n|summary|               pcv|\n+-------+------------------+\n|  count|               400|\n|   mean|38.884498480243195|\n| stddev| 8.151081380700974|\n|    min|               \t43|\n|    max|                 9|\n+-------+------------------+\n\n+-------+------------------+\n|summary|              wbcc|\n+-------+------------------+\n|  count|               400|\n|   mean| 8406.122448979597|\n| stddev|2523.2199758633965|\n|    min|             \t6200|\n|    max|              9900|\n+-------+------------------+\n\n+-------+------------------+\n|summary|              rbcc|\n+-------+------------------+\n|  count|               400|\n|   mean| 4.707434944237918|\n| stddev|0.8403143136011171|\n|    min|               2.1|\n|    max|                 8|\n+-------+------------------+\n\n+-------+----+\n|summary| htn|\n+-------+----+\n|  count| 400|\n|   mean|null|\n| stddev|null|\n|    min|  no|\n|    max| yes|\n+-------+----+\n\n+-------+----+\n|summary|  dm|\n+-------+----+\n|  count| 400|\n|   mean|null|\n| stddev|null|\n|    min| yes|\n|    max| yes|\n+-------+----+\n\n+-------+----+\n|summary| cad|\n+-------+----+\n|  count| 400|\n|   mean|null|\n| stddev|null|\n|    min|  no|\n|    max| yes|\n+-------+----+\n\n+-------+-----+\n|summary|appet|\n+-------+-----+\n|  count|  400|\n|   mean| null|\n| stddev| null|\n|    min| good|\n|    max| poor|\n+-------+-----+\n\n+-------+----+\n|summary|  pe|\n+-------+----+\n|  count| 400|\n|   mean|null|\n| stddev|null|\n|    min|good|\n|    max| yes|\n+-------+----+\n\n+-------+----+\n|summary| ane|\n+-------+----+\n|  count| 400|\n|   mean|null|\n| stddev|null|\n|    min|  no|\n|    max| yes|\n+-------+----+\n\n+-------+------+\n|summary| class|\n+-------+------+\n|  count|   400|\n|   mean|  null|\n| stddev|  null|\n|    min|   ckd|\n|    max|notckd|\n+-------+------+\n\n"
                }
            ], 
            "execution_count": 40
        }, 
        {
            "source": "from pyspark.ml.feature import StringIndexer\n\n\nindexer = StringIndexer(inputCol=\"class\", outputCol=\"label\")\nindexed = indexer.fit(df).transform(df)\nindexed.show()\nindexed.select('label').distinct().show()", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "+---+-----------------+------------------+---+---+--------+--------+----------+----------+-----------------+---+---+------------------+-----------------+----+-----------------+-----------------+-----------------+---+---+---+-----+---+---+-----+-----+\n|age|               bp|                sg| al| su|     rbc|      pc|       pcc|        ba|              bgr| bu| sc|               sod|              pot|hemo|              pcv|             wbcc|             rbcc|htn| dm|cad|appet| pe|ane|class|label|\n+---+-----------------+------------------+---+---+--------+--------+----------+----------+-----------------+---+---+------------------+-----------------+----+-----------------+-----------------+-----------------+---+---+---+-----+---+---+-----+-----+\n| 48|               80|              1.02|  1|  0|  normal|  normal|notpresent|notpresent|              121| 36|1.2|137.52875399361022|4.627243589743592|15.4|               44|             7800|              5.2|yes|yes| no| good| no| no|  ckd|  0.0|\n|  7|               50|              1.02|  4|  0|  normal|  normal|notpresent|notpresent|148.0365168539326| 18|0.8|137.52875399361022|4.627243589743592|11.3|               38|             6000|4.707434944237919| no| no| no| good| no| no|  ckd|  0.0|\n| 62|               80|              1.01|  2|  3|  normal|  normal|notpresent|notpresent|              423| 53|1.8|137.52875399361022|4.627243589743592| 9.6|               31|             7500|4.707434944237919| no|yes| no| poor| no|yes|  ckd|  0.0|\n| 48|               70|             1.005|  4|  0|  normal|abnormal|   present|notpresent|              117| 56|3.8|               111|              2.5|11.2|               32|             6700|              3.9|yes| no| no| poor|yes|yes|  ckd|  0.0|\n| 51|               80|              1.01|  2|  0|  normal|  normal|notpresent|notpresent|              106| 26|1.4|137.52875399361022|4.627243589743592|11.6|               35|             7300|              4.6| no| no| no| good| no| no|  ckd|  0.0|\n| 60|               90|             1.015|  3|  0|  normal|  normal|notpresent|notpresent|               74| 25|1.1|               142|              3.2|12.2|               39|             7800|              4.4|yes|yes| no| good|yes| no|  ckd|  0.0|\n| 68|               70|              1.01|  0|  0|  normal|  normal|notpresent|notpresent|              100| 54| 24|               104|                4|12.4|               36|8406.122448979591|4.707434944237919| no| no| no| good| no| no|  ckd|  0.0|\n| 24|76.46907216494846|             1.015|  2|  4|  normal|abnormal|notpresent|notpresent|              410| 31|1.1|137.52875399361022|4.627243589743592|12.4|               44|             6900|                5| no|yes| no| good|yes| no|  ckd|  0.0|\n| 52|              100|             1.015|  3|  0|  normal|abnormal|   present|notpresent|              138| 60|1.9|137.52875399361022|4.627243589743592|10.8|               33|             9600|                4|yes|yes| no| good| no|yes|  ckd|  0.0|\n| 53|               90|              1.02|  2|  0|abnormal|abnormal|   present|notpresent|               70|107|7.2|               114|              3.7| 9.5|               29|            12100|              3.7|yes|yes| no| poor| no|yes|  ckd|  0.0|\n| 50|               60|              1.01|  2|  4|  normal|abnormal|   present|notpresent|              490| 55|  4|137.52875399361022|4.627243589743592| 9.4|               28|8406.122448979591|4.707434944237919|yes|yes| no| good| no|yes|  ckd|  0.0|\n| 63|               70|              1.01|  3|  0|abnormal|abnormal|   present|notpresent|              380| 60|2.7|               131|              4.2|10.8|               32|             4500|              3.8|yes|yes| no| poor|yes| no|  ckd|  0.0|\n| 68|               70|             1.015|  3|  1|  normal|  normal|   present|notpresent|              208| 72|2.1|               138|              5.8| 9.7|               28|            12200|              3.4|yes|yes|yes| poor|yes| no|  ckd|  0.0|\n| 68|               70|1.0174079320113256|  0|  0|  normal|  normal|notpresent|notpresent|               98| 86|4.6|               135|              3.4| 9.8|38.88449848024316|8406.122448979591|4.707434944237919|yes|yes|yes| poor|yes| no|  ckd|  0.0|\n| 68|               80|              1.01|  3|  2|  normal|abnormal|   present|   present|              157| 90|4.1|               130|              6.4| 5.6|               16|            11000|              2.6|yes|yes|yes| poor|yes| no|  ckd|  0.0|\n| 40|               80|             1.015|  3|  0|  normal|  normal|notpresent|notpresent|               76|162|9.6|               141|              4.9| 7.6|               24|             3800|              2.8|yes| no| no| good| no|yes|  ckd|  0.0|\n| 47|               70|             1.015|  2|  0|  normal|  normal|notpresent|notpresent|               99| 46|2.2|               138|              4.1|12.6|38.88449848024316|8406.122448979591|4.707434944237919| no| no| no| good| no| no|  ckd|  0.0|\n| 47|               80|1.0174079320113256|  0|  0|  normal|  normal|notpresent|notpresent|              114| 87|5.2|               139|              3.7|12.1|38.88449848024316|8406.122448979591|4.707434944237919|yes| no| no| poor| no| no|  ckd|  0.0|\n| 60|              100|             1.025|  0|  3|  normal|  normal|notpresent|notpresent|              263| 27|1.3|               135|              4.3|12.7|               37|            11400|              4.3|yes|yes|yes| good| no| no|  ckd|  0.0|\n| 62|               60|             1.015|  1|  0|  normal|abnormal|   present|notpresent|              100| 31|1.6|137.52875399361022|4.627243589743592|10.3|               30|             5300|              3.7|yes| no|yes| good| no| no|  ckd|  0.0|\n+---+-----------------+------------------+---+---+--------+--------+----------+----------+-----------------+---+---+------------------+-----------------+----+-----------------+-----------------+-----------------+---+---+---+-----+---+---+-----+-----+\nonly showing top 20 rows\n\n+-----+\n|label|\n+-----+\n|  0.0|\n|  1.0|\n+-----+\n\n"
                }
            ], 
            "execution_count": 41
        }, 
        {
            "source": "#Count the value of null in every column\nfor col in df.columns:\n    print(col, \"\\t\", \"with null values: \", df.filter(df[col].isNull()).count())", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "age \t with null values:  0\nbp \t with null values:  0\nsg \t with null values:  0\nal \t with null values:  0\nsu \t with null values:  0\nrbc \t with null values:  0\npc \t with null values:  0\npcc \t with null values:  0\nba \t with null values:  0\nbgr \t with null values:  0\nbu \t with null values:  0\nsc \t with null values:  0\nsod \t with null values:  0\npot \t with null values:  0\nhemo \t with null values:  0\npcv \t with null values:  0\nwbcc \t with null values:  0\nrbcc \t with null values:  0\nhtn \t with null values:  0\ndm \t with null values:  0\ncad \t with null values:  0\nappet \t with null values:  0\npe \t with null values:  0\nane \t with null values:  0\nclass \t with null values:  0\n"
                }
            ], 
            "execution_count": 42
        }, 
        {
            "source": "from pyspark.ml.feature import StringIndexer\n\ncat_col_list=['al', 'su', 'rbc', 'pc', 'pcc', 'ba', 'htn', 'dm', 'cad', 'appet', 'pe', 'ane']\nfor col in cat_col_list:\n    indexer = StringIndexer(inputCol=col, outputCol=col + \"Index\")\n    indexed = indexer.fit(indexed).transform(indexed)\nindexed.show()\n\n", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "+---+-----------------+------------------+---+---+--------+--------+----------+----------+-----------------+---+---+------------------+-----------------+----+-----------------+-----------------+-----------------+---+---+---+-----+---+---+-----+-----+-------+-------+--------+-------+--------+-------+--------+-------+--------+----------+-------+--------+\n|age|               bp|                sg| al| su|     rbc|      pc|       pcc|        ba|              bgr| bu| sc|               sod|              pot|hemo|              pcv|             wbcc|             rbcc|htn| dm|cad|appet| pe|ane|class|label|alIndex|suIndex|rbcIndex|pcIndex|pccIndex|baIndex|htnIndex|dmIndex|cadIndex|appetIndex|peIndex|aneIndex|\n+---+-----------------+------------------+---+---+--------+--------+----------+----------+-----------------+---+---+------------------+-----------------+----+-----------------+-----------------+-----------------+---+---+---+-----+---+---+-----+-----+-------+-------+--------+-------+--------+-------+--------+-------+--------+----------+-------+--------+\n| 48|               80|              1.02|  1|  0|  normal|  normal|notpresent|notpresent|              121| 36|1.2|137.52875399361022|4.627243589743592|15.4|               44|             7800|              5.2|yes|yes| no| good| no| no|  ckd|  0.0|    1.0|    0.0|     0.0|    0.0|     0.0|    0.0|     1.0|    1.0|     0.0|       0.0|    0.0|     0.0|\n|  7|               50|              1.02|  4|  0|  normal|  normal|notpresent|notpresent|148.0365168539326| 18|0.8|137.52875399361022|4.627243589743592|11.3|               38|             6000|4.707434944237919| no| no| no| good| no| no|  ckd|  0.0|    4.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|       0.0|    0.0|     0.0|\n| 62|               80|              1.01|  2|  3|  normal|  normal|notpresent|notpresent|              423| 53|1.8|137.52875399361022|4.627243589743592| 9.6|               31|             7500|4.707434944237919| no|yes| no| poor| no|yes|  ckd|  0.0|    2.0|    2.0|     0.0|    0.0|     0.0|    0.0|     0.0|    1.0|     0.0|       1.0|    0.0|     1.0|\n| 48|               70|             1.005|  4|  0|  normal|abnormal|   present|notpresent|              117| 56|3.8|               111|              2.5|11.2|               32|             6700|              3.9|yes| no| no| poor|yes|yes|  ckd|  0.0|    4.0|    0.0|     0.0|    1.0|     1.0|    0.0|     1.0|    0.0|     0.0|       1.0|    1.0|     1.0|\n| 51|               80|              1.01|  2|  0|  normal|  normal|notpresent|notpresent|              106| 26|1.4|137.52875399361022|4.627243589743592|11.6|               35|             7300|              4.6| no| no| no| good| no| no|  ckd|  0.0|    2.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|       0.0|    0.0|     0.0|\n| 60|               90|             1.015|  3|  0|  normal|  normal|notpresent|notpresent|               74| 25|1.1|               142|              3.2|12.2|               39|             7800|              4.4|yes|yes| no| good|yes| no|  ckd|  0.0|    3.0|    0.0|     0.0|    0.0|     0.0|    0.0|     1.0|    1.0|     0.0|       0.0|    1.0|     0.0|\n| 68|               70|              1.01|  0|  0|  normal|  normal|notpresent|notpresent|              100| 54| 24|               104|                4|12.4|               36|8406.122448979591|4.707434944237919| no| no| no| good| no| no|  ckd|  0.0|    0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|       0.0|    0.0|     0.0|\n| 24|76.46907216494846|             1.015|  2|  4|  normal|abnormal|notpresent|notpresent|              410| 31|1.1|137.52875399361022|4.627243589743592|12.4|               44|             6900|                5| no|yes| no| good|yes| no|  ckd|  0.0|    2.0|    3.0|     0.0|    1.0|     0.0|    0.0|     0.0|    1.0|     0.0|       0.0|    1.0|     0.0|\n| 52|              100|             1.015|  3|  0|  normal|abnormal|   present|notpresent|              138| 60|1.9|137.52875399361022|4.627243589743592|10.8|               33|             9600|                4|yes|yes| no| good| no|yes|  ckd|  0.0|    3.0|    0.0|     0.0|    1.0|     1.0|    0.0|     1.0|    1.0|     0.0|       0.0|    0.0|     1.0|\n| 53|               90|              1.02|  2|  0|abnormal|abnormal|   present|notpresent|               70|107|7.2|               114|              3.7| 9.5|               29|            12100|              3.7|yes|yes| no| poor| no|yes|  ckd|  0.0|    2.0|    0.0|     1.0|    1.0|     1.0|    0.0|     1.0|    1.0|     0.0|       1.0|    0.0|     1.0|\n| 50|               60|              1.01|  2|  4|  normal|abnormal|   present|notpresent|              490| 55|  4|137.52875399361022|4.627243589743592| 9.4|               28|8406.122448979591|4.707434944237919|yes|yes| no| good| no|yes|  ckd|  0.0|    2.0|    3.0|     0.0|    1.0|     1.0|    0.0|     1.0|    1.0|     0.0|       0.0|    0.0|     1.0|\n| 63|               70|              1.01|  3|  0|abnormal|abnormal|   present|notpresent|              380| 60|2.7|               131|              4.2|10.8|               32|             4500|              3.8|yes|yes| no| poor|yes| no|  ckd|  0.0|    3.0|    0.0|     1.0|    1.0|     1.0|    0.0|     1.0|    1.0|     0.0|       1.0|    1.0|     0.0|\n| 68|               70|             1.015|  3|  1|  normal|  normal|   present|notpresent|              208| 72|2.1|               138|              5.8| 9.7|               28|            12200|              3.4|yes|yes|yes| poor|yes| no|  ckd|  0.0|    3.0|    4.0|     0.0|    0.0|     1.0|    0.0|     1.0|    1.0|     1.0|       1.0|    1.0|     0.0|\n| 68|               70|1.0174079320113256|  0|  0|  normal|  normal|notpresent|notpresent|               98| 86|4.6|               135|              3.4| 9.8|38.88449848024316|8406.122448979591|4.707434944237919|yes|yes|yes| poor|yes| no|  ckd|  0.0|    0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     1.0|    1.0|     1.0|       1.0|    1.0|     0.0|\n| 68|               80|              1.01|  3|  2|  normal|abnormal|   present|   present|              157| 90|4.1|               130|              6.4| 5.6|               16|            11000|              2.6|yes|yes|yes| poor|yes| no|  ckd|  0.0|    3.0|    1.0|     0.0|    1.0|     1.0|    1.0|     1.0|    1.0|     1.0|       1.0|    1.0|     0.0|\n| 40|               80|             1.015|  3|  0|  normal|  normal|notpresent|notpresent|               76|162|9.6|               141|              4.9| 7.6|               24|             3800|              2.8|yes| no| no| good| no|yes|  ckd|  0.0|    3.0|    0.0|     0.0|    0.0|     0.0|    0.0|     1.0|    0.0|     0.0|       0.0|    0.0|     1.0|\n| 47|               70|             1.015|  2|  0|  normal|  normal|notpresent|notpresent|               99| 46|2.2|               138|              4.1|12.6|38.88449848024316|8406.122448979591|4.707434944237919| no| no| no| good| no| no|  ckd|  0.0|    2.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|       0.0|    0.0|     0.0|\n| 47|               80|1.0174079320113256|  0|  0|  normal|  normal|notpresent|notpresent|              114| 87|5.2|               139|              3.7|12.1|38.88449848024316|8406.122448979591|4.707434944237919|yes| no| no| poor| no| no|  ckd|  0.0|    0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     1.0|    0.0|     0.0|       1.0|    0.0|     0.0|\n| 60|              100|             1.025|  0|  3|  normal|  normal|notpresent|notpresent|              263| 27|1.3|               135|              4.3|12.7|               37|            11400|              4.3|yes|yes|yes| good| no| no|  ckd|  0.0|    0.0|    2.0|     0.0|    0.0|     0.0|    0.0|     1.0|    1.0|     1.0|       0.0|    0.0|     0.0|\n| 62|               60|             1.015|  1|  0|  normal|abnormal|   present|notpresent|              100| 31|1.6|137.52875399361022|4.627243589743592|10.3|               30|             5300|              3.7|yes| no|yes| good| no| no|  ckd|  0.0|    1.0|    0.0|     0.0|    1.0|     1.0|    0.0|     1.0|    0.0|     1.0|       0.0|    0.0|     0.0|\n+---+-----------------+------------------+---+---+--------+--------+----------+----------+-----------------+---+---+------------------+-----------------+----+-----------------+-----------------+-----------------+---+---+---+-----+---+---+-----+-----+-------+-------+--------+-------+--------+-------+--------+-------+--------+----------+-------+--------+\nonly showing top 20 rows\n\n"
                }
            ], 
            "execution_count": 43
        }, 
        {
            "source": "numeric_cols=['age', 'bp', 'sg','bgr', 'bu', 'sc', 'sod', 'pot', 'hemo', 'pcv', 'wbcc', 'rbcc']\nindex_cat_col_list=['alIndex', 'suIndex', 'rbcIndex', 'pcIndex', 'pccIndex', 'baIndex', 'htnIndex', 'dmIndex', 'cadIndex', 'appetIndex', 'peIndex', 'aneIndex']\n\nfeatures_list=numeric_cols + index_cat_col_list\nfeatures_list\n", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 44, 
                    "output_type": "execute_result", 
                    "data": {
                        "text/plain": "['age',\n 'bp',\n 'sg',\n 'bgr',\n 'bu',\n 'sc',\n 'sod',\n 'pot',\n 'hemo',\n 'pcv',\n 'wbcc',\n 'rbcc',\n 'alIndex',\n 'suIndex',\n 'rbcIndex',\n 'pcIndex',\n 'pccIndex',\n 'baIndex',\n 'htnIndex',\n 'dmIndex',\n 'cadIndex',\n 'appetIndex',\n 'peIndex',\n 'aneIndex']"
                    }, 
                    "metadata": {}
                }
            ], 
            "execution_count": 44
        }, 
        {
            "source": "from pyspark.sql.types import DoubleType\n\nnumeric_cols=['age', 'bp', 'sg','bgr', 'bu', 'sc', 'sod', 'pot', 'hemo', 'pcv', 'wbcc', 'rbcc']\nfor col in numeric_cols:\n    indexed = indexed.withColumn(col, indexed[col].cast(DoubleType()))", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 45
        }, 
        {
            "source": "from pyspark.ml.linalg import Vectors\nfrom pyspark.ml.feature import VectorAssembler\n\nvectorAssembler = VectorAssembler(inputCols=features_list,\n                                  outputCol=\"features\")\n\n# For your special case that has string instead of doubles you should cast them first.\n# expr = [col(c).cast(\"Double\").alias(c) \n#         for c in vectorAssembler.getInputCols()]\n\n# df2 = df2.select(*expr)\nfeatures_vectorized = vectorAssembler.transform(indexed)\nfeatures_vectorized.show()", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "+----+-----------------+------------------+---+---+--------+--------+----------+----------+-----------------+-----+----+------------------+-----------------+----+-----------------+-----------------+-----------------+---+---+---+-----+---+---+-----+-----+-------+-------+--------+-------+--------+-------+--------+-------+--------+----------+-------+--------+--------------------+\n| age|               bp|                sg| al| su|     rbc|      pc|       pcc|        ba|              bgr|   bu|  sc|               sod|              pot|hemo|              pcv|             wbcc|             rbcc|htn| dm|cad|appet| pe|ane|class|label|alIndex|suIndex|rbcIndex|pcIndex|pccIndex|baIndex|htnIndex|dmIndex|cadIndex|appetIndex|peIndex|aneIndex|            features|\n+----+-----------------+------------------+---+---+--------+--------+----------+----------+-----------------+-----+----+------------------+-----------------+----+-----------------+-----------------+-----------------+---+---+---+-----+---+---+-----+-----+-------+-------+--------+-------+--------+-------+--------+-------+--------+----------+-------+--------+--------------------+\n|48.0|             80.0|              1.02|  1|  0|  normal|  normal|notpresent|notpresent|            121.0| 36.0| 1.2|137.52875399361022|4.627243589743592|15.4|             44.0|           7800.0|              5.2|yes|yes| no| good| no| no|  ckd|  0.0|    1.0|    0.0|     0.0|    0.0|     0.0|    0.0|     1.0|    1.0|     0.0|       0.0|    0.0|     0.0|[48.0,80.0,1.02,1...|\n| 7.0|             50.0|              1.02|  4|  0|  normal|  normal|notpresent|notpresent|148.0365168539326| 18.0| 0.8|137.52875399361022|4.627243589743592|11.3|             38.0|           6000.0|4.707434944237919| no| no| no| good| no| no|  ckd|  0.0|    4.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|       0.0|    0.0|     0.0|(24,[0,1,2,3,4,5,...|\n|62.0|             80.0|              1.01|  2|  3|  normal|  normal|notpresent|notpresent|            423.0| 53.0| 1.8|137.52875399361022|4.627243589743592| 9.6|             31.0|           7500.0|4.707434944237919| no|yes| no| poor| no|yes|  ckd|  0.0|    2.0|    2.0|     0.0|    0.0|     0.0|    0.0|     0.0|    1.0|     0.0|       1.0|    0.0|     1.0|[62.0,80.0,1.01,4...|\n|48.0|             70.0|             1.005|  4|  0|  normal|abnormal|   present|notpresent|            117.0| 56.0| 3.8|             111.0|              2.5|11.2|             32.0|           6700.0|              3.9|yes| no| no| poor|yes|yes|  ckd|  0.0|    4.0|    0.0|     0.0|    1.0|     1.0|    0.0|     1.0|    0.0|     0.0|       1.0|    1.0|     1.0|[48.0,70.0,1.005,...|\n|51.0|             80.0|              1.01|  2|  0|  normal|  normal|notpresent|notpresent|            106.0| 26.0| 1.4|137.52875399361022|4.627243589743592|11.6|             35.0|           7300.0|              4.6| no| no| no| good| no| no|  ckd|  0.0|    2.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|       0.0|    0.0|     0.0|(24,[0,1,2,3,4,5,...|\n|60.0|             90.0|             1.015|  3|  0|  normal|  normal|notpresent|notpresent|             74.0| 25.0| 1.1|             142.0|              3.2|12.2|             39.0|           7800.0|              4.4|yes|yes| no| good|yes| no|  ckd|  0.0|    3.0|    0.0|     0.0|    0.0|     0.0|    0.0|     1.0|    1.0|     0.0|       0.0|    1.0|     0.0|[60.0,90.0,1.015,...|\n|68.0|             70.0|              1.01|  0|  0|  normal|  normal|notpresent|notpresent|            100.0| 54.0|24.0|             104.0|              4.0|12.4|             36.0|8406.122448979591|4.707434944237919| no| no| no| good| no| no|  ckd|  0.0|    0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|       0.0|    0.0|     0.0|(24,[0,1,2,3,4,5,...|\n|24.0|76.46907216494846|             1.015|  2|  4|  normal|abnormal|notpresent|notpresent|            410.0| 31.0| 1.1|137.52875399361022|4.627243589743592|12.4|             44.0|           6900.0|              5.0| no|yes| no| good|yes| no|  ckd|  0.0|    2.0|    3.0|     0.0|    1.0|     0.0|    0.0|     0.0|    1.0|     0.0|       0.0|    1.0|     0.0|[24.0,76.46907216...|\n|52.0|            100.0|             1.015|  3|  0|  normal|abnormal|   present|notpresent|            138.0| 60.0| 1.9|137.52875399361022|4.627243589743592|10.8|             33.0|           9600.0|              4.0|yes|yes| no| good| no|yes|  ckd|  0.0|    3.0|    0.0|     0.0|    1.0|     1.0|    0.0|     1.0|    1.0|     0.0|       0.0|    0.0|     1.0|[52.0,100.0,1.015...|\n|53.0|             90.0|              1.02|  2|  0|abnormal|abnormal|   present|notpresent|             70.0|107.0| 7.2|             114.0|              3.7| 9.5|             29.0|          12100.0|              3.7|yes|yes| no| poor| no|yes|  ckd|  0.0|    2.0|    0.0|     1.0|    1.0|     1.0|    0.0|     1.0|    1.0|     0.0|       1.0|    0.0|     1.0|[53.0,90.0,1.02,7...|\n|50.0|             60.0|              1.01|  2|  4|  normal|abnormal|   present|notpresent|            490.0| 55.0| 4.0|137.52875399361022|4.627243589743592| 9.4|             28.0|8406.122448979591|4.707434944237919|yes|yes| no| good| no|yes|  ckd|  0.0|    2.0|    3.0|     0.0|    1.0|     1.0|    0.0|     1.0|    1.0|     0.0|       0.0|    0.0|     1.0|[50.0,60.0,1.01,4...|\n|63.0|             70.0|              1.01|  3|  0|abnormal|abnormal|   present|notpresent|            380.0| 60.0| 2.7|             131.0|              4.2|10.8|             32.0|           4500.0|              3.8|yes|yes| no| poor|yes| no|  ckd|  0.0|    3.0|    0.0|     1.0|    1.0|     1.0|    0.0|     1.0|    1.0|     0.0|       1.0|    1.0|     0.0|[63.0,70.0,1.01,3...|\n|68.0|             70.0|             1.015|  3|  1|  normal|  normal|   present|notpresent|            208.0| 72.0| 2.1|             138.0|              5.8| 9.7|             28.0|          12200.0|              3.4|yes|yes|yes| poor|yes| no|  ckd|  0.0|    3.0|    4.0|     0.0|    0.0|     1.0|    0.0|     1.0|    1.0|     1.0|       1.0|    1.0|     0.0|[68.0,70.0,1.015,...|\n|68.0|             70.0|1.0174079320113256|  0|  0|  normal|  normal|notpresent|notpresent|             98.0| 86.0| 4.6|             135.0|              3.4| 9.8|38.88449848024316|8406.122448979591|4.707434944237919|yes|yes|yes| poor|yes| no|  ckd|  0.0|    0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     1.0|    1.0|     1.0|       1.0|    1.0|     0.0|[68.0,70.0,1.0174...|\n|68.0|             80.0|              1.01|  3|  2|  normal|abnormal|   present|   present|            157.0| 90.0| 4.1|             130.0|              6.4| 5.6|             16.0|          11000.0|              2.6|yes|yes|yes| poor|yes| no|  ckd|  0.0|    3.0|    1.0|     0.0|    1.0|     1.0|    1.0|     1.0|    1.0|     1.0|       1.0|    1.0|     0.0|[68.0,80.0,1.01,1...|\n|40.0|             80.0|             1.015|  3|  0|  normal|  normal|notpresent|notpresent|             76.0|162.0| 9.6|             141.0|              4.9| 7.6|             24.0|           3800.0|              2.8|yes| no| no| good| no|yes|  ckd|  0.0|    3.0|    0.0|     0.0|    0.0|     0.0|    0.0|     1.0|    0.0|     0.0|       0.0|    0.0|     1.0|[40.0,80.0,1.015,...|\n|47.0|             70.0|             1.015|  2|  0|  normal|  normal|notpresent|notpresent|             99.0| 46.0| 2.2|             138.0|              4.1|12.6|38.88449848024316|8406.122448979591|4.707434944237919| no| no| no| good| no| no|  ckd|  0.0|    2.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|       0.0|    0.0|     0.0|(24,[0,1,2,3,4,5,...|\n|47.0|             80.0|1.0174079320113256|  0|  0|  normal|  normal|notpresent|notpresent|            114.0| 87.0| 5.2|             139.0|              3.7|12.1|38.88449848024316|8406.122448979591|4.707434944237919|yes| no| no| poor| no| no|  ckd|  0.0|    0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     1.0|    0.0|     0.0|       1.0|    0.0|     0.0|(24,[0,1,2,3,4,5,...|\n|60.0|            100.0|             1.025|  0|  3|  normal|  normal|notpresent|notpresent|            263.0| 27.0| 1.3|             135.0|              4.3|12.7|             37.0|          11400.0|              4.3|yes|yes|yes| good| no| no|  ckd|  0.0|    0.0|    2.0|     0.0|    0.0|     0.0|    0.0|     1.0|    1.0|     1.0|       0.0|    0.0|     0.0|[60.0,100.0,1.025...|\n|62.0|             60.0|             1.015|  1|  0|  normal|abnormal|   present|notpresent|            100.0| 31.0| 1.6|137.52875399361022|4.627243589743592|10.3|             30.0|           5300.0|              3.7|yes| no|yes| good| no| no|  ckd|  0.0|    1.0|    0.0|     0.0|    1.0|     1.0|    0.0|     1.0|    0.0|     1.0|       0.0|    0.0|     0.0|[62.0,60.0,1.015,...|\n+----+-----------------+------------------+---+---+--------+--------+----------+----------+-----------------+-----+----+------------------+-----------------+----+-----------------+-----------------+-----------------+---+---+---+-----+---+---+-----+-----+-------+-------+--------+-------+--------+-------+--------+-------+--------+----------+-------+--------+--------------------+\nonly showing top 20 rows\n\n"
                }
            ], 
            "execution_count": 46
        }, 
        {
            "source": "from pyspark.ml.feature import Normalizer\nfrom pyspark.ml.linalg import Vectors\n\nnormalizer = Normalizer(inputCol=\"features\", outputCol=\"features_norm\", p=1.0)\nl1NormData = normalizer.transform(features_vectorized)\nl1NormData.show()", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "+----+-----------------+------------------+---+---+--------+--------+----------+----------+-----------------+-----+----+------------------+-----------------+----+-----------------+-----------------+-----------------+---+---+---+-----+---+---+-----+-----+-------+-------+--------+-------+--------+-------+--------+-------+--------+----------+-------+--------+--------------------+--------------------+\n| age|               bp|                sg| al| su|     rbc|      pc|       pcc|        ba|              bgr|   bu|  sc|               sod|              pot|hemo|              pcv|             wbcc|             rbcc|htn| dm|cad|appet| pe|ane|class|label|alIndex|suIndex|rbcIndex|pcIndex|pccIndex|baIndex|htnIndex|dmIndex|cadIndex|appetIndex|peIndex|aneIndex|            features|       features_norm|\n+----+-----------------+------------------+---+---+--------+--------+----------+----------+-----------------+-----+----+------------------+-----------------+----+-----------------+-----------------+-----------------+---+---+---+-----+---+---+-----+-----+-------+-------+--------+-------+--------+-------+--------+-------+--------+----------+-------+--------+--------------------+--------------------+\n|48.0|             80.0|              1.02|  1|  0|  normal|  normal|notpresent|notpresent|            121.0| 36.0| 1.2|137.52875399361022|4.627243589743592|15.4|             44.0|           7800.0|              5.2|yes|yes| no| good| no| no|  ckd|  0.0|    1.0|    0.0|     0.0|    0.0|     0.0|    0.0|     1.0|    1.0|     0.0|       0.0|    0.0|     0.0|[48.0,80.0,1.02,1...|[0.00578524031092...|\n| 7.0|             50.0|              1.02|  4|  0|  normal|  normal|notpresent|notpresent|148.0365168539326| 18.0| 0.8|137.52875399361022|4.627243589743592|11.3|             38.0|           6000.0|4.707434944237919| no| no| no| good| no| no|  ckd|  0.0|    4.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|       0.0|    0.0|     0.0|(24,[0,1,2,3,4,5,...|(24,[0,1,2,3,4,5,...|\n|62.0|             80.0|              1.01|  2|  3|  normal|  normal|notpresent|notpresent|            423.0| 53.0| 1.8|137.52875399361022|4.627243589743592| 9.6|             31.0|           7500.0|4.707434944237919| no|yes| no| poor| no|yes|  ckd|  0.0|    2.0|    2.0|     0.0|    0.0|     0.0|    0.0|     0.0|    1.0|     0.0|       1.0|    0.0|     1.0|[62.0,80.0,1.01,4...|[0.00745615889881...|\n|48.0|             70.0|             1.005|  4|  0|  normal|abnormal|   present|notpresent|            117.0| 56.0| 3.8|             111.0|              2.5|11.2|             32.0|           6700.0|              3.9|yes| no| no| poor|yes|yes|  ckd|  0.0|    4.0|    0.0|     0.0|    1.0|     1.0|    0.0|     1.0|    0.0|     0.0|       1.0|    1.0|     1.0|[48.0,70.0,1.005,...|[0.00669791897052...|\n|51.0|             80.0|              1.01|  2|  0|  normal|  normal|notpresent|notpresent|            106.0| 26.0| 1.4|137.52875399361022|4.627243589743592|11.6|             35.0|           7300.0|              4.6| no| no| no| good| no| no|  ckd|  0.0|    2.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|       0.0|    0.0|     0.0|(24,[0,1,2,3,4,5,...|(24,[0,1,2,3,4,5,...|\n|60.0|             90.0|             1.015|  3|  0|  normal|  normal|notpresent|notpresent|             74.0| 25.0| 1.1|             142.0|              3.2|12.2|             39.0|           7800.0|              4.4|yes|yes| no| good|yes| no|  ckd|  0.0|    3.0|    0.0|     0.0|    0.0|     0.0|    0.0|     1.0|    1.0|     0.0|       0.0|    1.0|     0.0|[60.0,90.0,1.015,...|[0.00726575654992...|\n|68.0|             70.0|              1.01|  0|  0|  normal|  normal|notpresent|notpresent|            100.0| 54.0|24.0|             104.0|              4.0|12.4|             36.0|8406.122448979591|4.707434944237919| no| no| no| good| no| no|  ckd|  0.0|    0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|       0.0|    0.0|     0.0|(24,[0,1,2,3,4,5,...|(24,[0,1,2,3,4,5,...|\n|24.0|76.46907216494846|             1.015|  2|  4|  normal|abnormal|notpresent|notpresent|            410.0| 31.0| 1.1|137.52875399361022|4.627243589743592|12.4|             44.0|           6900.0|              5.0| no|yes| no| good|yes| no|  ckd|  0.0|    2.0|    3.0|     0.0|    1.0|     0.0|    0.0|     0.0|    1.0|     0.0|       0.0|    1.0|     0.0|[24.0,76.46907216...|[0.00313514838152...|\n|52.0|            100.0|             1.015|  3|  0|  normal|abnormal|   present|notpresent|            138.0| 60.0| 1.9|137.52875399361022|4.627243589743592|10.8|             33.0|           9600.0|              4.0|yes|yes| no| good| no|yes|  ckd|  0.0|    3.0|    0.0|     0.0|    1.0|     1.0|    0.0|     1.0|    1.0|     0.0|       0.0|    0.0|     1.0|[52.0,100.0,1.015...|[0.00512271311618...|\n|53.0|             90.0|              1.02|  2|  0|abnormal|abnormal|   present|notpresent|             70.0|107.0| 7.2|             114.0|              3.7| 9.5|             29.0|          12100.0|              3.7|yes|yes| no| poor| no|yes|  ckd|  0.0|    2.0|    0.0|     1.0|    1.0|     1.0|    0.0|     1.0|    1.0|     0.0|       1.0|    0.0|     1.0|[53.0,90.0,1.02,7...|[0.00420731087740...|\n|50.0|             60.0|              1.01|  2|  4|  normal|abnormal|   present|notpresent|            490.0| 55.0| 4.0|137.52875399361022|4.627243589743592| 9.4|             28.0|8406.122448979591|4.707434944237919|yes|yes| no| good| no|yes|  ckd|  0.0|    2.0|    3.0|     0.0|    1.0|     1.0|    0.0|     1.0|    1.0|     0.0|       0.0|    0.0|     1.0|[50.0,60.0,1.01,4...|[0.00539933720326...|\n|63.0|             70.0|              1.01|  3|  0|abnormal|abnormal|   present|notpresent|            380.0| 60.0| 2.7|             131.0|              4.2|10.8|             32.0|           4500.0|              3.8|yes|yes| no| poor|yes| no|  ckd|  0.0|    3.0|    0.0|     1.0|    1.0|     1.0|    0.0|     1.0|    1.0|     0.0|       1.0|    1.0|     0.0|[63.0,70.0,1.01,3...|[0.01195784007243...|\n|68.0|             70.0|             1.015|  3|  1|  normal|  normal|   present|notpresent|            208.0| 72.0| 2.1|             138.0|              5.8| 9.7|             28.0|          12200.0|              3.4|yes|yes|yes| poor|yes| no|  ckd|  0.0|    3.0|    4.0|     0.0|    0.0|     1.0|    0.0|     1.0|    1.0|     1.0|       1.0|    1.0|     0.0|[68.0,70.0,1.015,...|[0.00530461973872...|\n|68.0|             70.0|1.0174079320113256|  0|  0|  normal|  normal|notpresent|notpresent|             98.0| 86.0| 4.6|             135.0|              3.4| 9.8|38.88449848024316|8406.122448979591|4.707434944237919|yes|yes|yes| poor|yes| no|  ckd|  0.0|    0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     1.0|    1.0|     1.0|       1.0|    1.0|     0.0|[68.0,70.0,1.0174...|[0.00761432819416...|\n|68.0|             80.0|              1.01|  3|  2|  normal|abnormal|   present|   present|            157.0| 90.0| 4.1|             130.0|              6.4| 5.6|             16.0|          11000.0|              2.6|yes|yes|yes| poor|yes| no|  ckd|  0.0|    3.0|    1.0|     0.0|    1.0|     1.0|    1.0|     1.0|    1.0|     1.0|       1.0|    1.0|     0.0|[68.0,80.0,1.01,1...|[0.00587589250918...|\n|40.0|             80.0|             1.015|  3|  0|  normal|  normal|notpresent|notpresent|             76.0|162.0| 9.6|             141.0|              4.9| 7.6|             24.0|           3800.0|              2.8|yes| no| no| good| no|yes|  ckd|  0.0|    3.0|    0.0|     0.0|    0.0|     0.0|    0.0|     1.0|    0.0|     0.0|       0.0|    0.0|     1.0|[40.0,80.0,1.015,...|[0.00918713387836...|\n|47.0|             70.0|             1.015|  2|  0|  normal|  normal|notpresent|notpresent|             99.0| 46.0| 2.2|             138.0|              4.1|12.6|38.88449848024316|8406.122448979591|4.707434944237919| no| no| no| good| no| no|  ckd|  0.0|    2.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     0.0|       0.0|    0.0|     0.0|(24,[0,1,2,3,4,5,...|(24,[0,1,2,3,4,5,...|\n|47.0|             80.0|1.0174079320113256|  0|  0|  normal|  normal|notpresent|notpresent|            114.0| 87.0| 5.2|             139.0|              3.7|12.1|38.88449848024316|8406.122448979591|4.707434944237919|yes| no| no| poor| no| no|  ckd|  0.0|    0.0|    0.0|     0.0|    0.0|     0.0|    0.0|     1.0|    0.0|     0.0|       1.0|    0.0|     0.0|(24,[0,1,2,3,4,5,...|(24,[0,1,2,3,4,5,...|\n|60.0|            100.0|             1.025|  0|  3|  normal|  normal|notpresent|notpresent|            263.0| 27.0| 1.3|             135.0|              4.3|12.7|             37.0|          11400.0|              4.3|yes|yes|yes| good| no| no|  ckd|  0.0|    0.0|    2.0|     0.0|    0.0|     0.0|    0.0|     1.0|    1.0|     1.0|       0.0|    0.0|     0.0|[60.0,100.0,1.025...|[0.00497899486541...|\n|62.0|             60.0|             1.015|  1|  0|  normal|abnormal|   present|notpresent|            100.0| 31.0| 1.6|137.52875399361022|4.627243589743592|10.3|             30.0|           5300.0|              3.7|yes| no|yes| good| no| no|  ckd|  0.0|    1.0|    0.0|     0.0|    1.0|     1.0|    0.0|     1.0|    0.0|     1.0|       0.0|    0.0|     0.0|[62.0,60.0,1.015,...|[0.01078866724045...|\n+----+-----------------+------------------+---+---+--------+--------+----------+----------+-----------------+-----+----+------------------+-----------------+----+-----------------+-----------------+-----------------+---+---+---+-----+---+---+-----+-----+-------+-------+--------+-------+--------+-------+--------+-------+--------+----------+-------+--------+--------------------+--------------------+\nonly showing top 20 rows\n\n"
                }
            ], 
            "execution_count": 47
        }, 
        {
            "source": "df_combine = l1NormData\nfor col in features_list:\n      df_combine = df_combine.drop(col)\n        \ndf_combine.show()        ", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "+---+---+--------+--------+----------+----------+---+---+---+-----+---+---+-----+-----+--------------------+--------------------+\n| al| su|     rbc|      pc|       pcc|        ba|htn| dm|cad|appet| pe|ane|class|label|            features|       features_norm|\n+---+---+--------+--------+----------+----------+---+---+---+-----+---+---+-----+-----+--------------------+--------------------+\n|  1|  0|  normal|  normal|notpresent|notpresent|yes|yes| no| good| no| no|  ckd|  0.0|[48.0,80.0,1.02,1...|[0.00578524031092...|\n|  4|  0|  normal|  normal|notpresent|notpresent| no| no| no| good| no| no|  ckd|  0.0|(24,[0,1,2,3,4,5,...|(24,[0,1,2,3,4,5,...|\n|  2|  3|  normal|  normal|notpresent|notpresent| no|yes| no| poor| no|yes|  ckd|  0.0|[62.0,80.0,1.01,4...|[0.00745615889881...|\n|  4|  0|  normal|abnormal|   present|notpresent|yes| no| no| poor|yes|yes|  ckd|  0.0|[48.0,70.0,1.005,...|[0.00669791897052...|\n|  2|  0|  normal|  normal|notpresent|notpresent| no| no| no| good| no| no|  ckd|  0.0|(24,[0,1,2,3,4,5,...|(24,[0,1,2,3,4,5,...|\n|  3|  0|  normal|  normal|notpresent|notpresent|yes|yes| no| good|yes| no|  ckd|  0.0|[60.0,90.0,1.015,...|[0.00726575654992...|\n|  0|  0|  normal|  normal|notpresent|notpresent| no| no| no| good| no| no|  ckd|  0.0|(24,[0,1,2,3,4,5,...|(24,[0,1,2,3,4,5,...|\n|  2|  4|  normal|abnormal|notpresent|notpresent| no|yes| no| good|yes| no|  ckd|  0.0|[24.0,76.46907216...|[0.00313514838152...|\n|  3|  0|  normal|abnormal|   present|notpresent|yes|yes| no| good| no|yes|  ckd|  0.0|[52.0,100.0,1.015...|[0.00512271311618...|\n|  2|  0|abnormal|abnormal|   present|notpresent|yes|yes| no| poor| no|yes|  ckd|  0.0|[53.0,90.0,1.02,7...|[0.00420731087740...|\n|  2|  4|  normal|abnormal|   present|notpresent|yes|yes| no| good| no|yes|  ckd|  0.0|[50.0,60.0,1.01,4...|[0.00539933720326...|\n|  3|  0|abnormal|abnormal|   present|notpresent|yes|yes| no| poor|yes| no|  ckd|  0.0|[63.0,70.0,1.01,3...|[0.01195784007243...|\n|  3|  1|  normal|  normal|   present|notpresent|yes|yes|yes| poor|yes| no|  ckd|  0.0|[68.0,70.0,1.015,...|[0.00530461973872...|\n|  0|  0|  normal|  normal|notpresent|notpresent|yes|yes|yes| poor|yes| no|  ckd|  0.0|[68.0,70.0,1.0174...|[0.00761432819416...|\n|  3|  2|  normal|abnormal|   present|   present|yes|yes|yes| poor|yes| no|  ckd|  0.0|[68.0,80.0,1.01,1...|[0.00587589250918...|\n|  3|  0|  normal|  normal|notpresent|notpresent|yes| no| no| good| no|yes|  ckd|  0.0|[40.0,80.0,1.015,...|[0.00918713387836...|\n|  2|  0|  normal|  normal|notpresent|notpresent| no| no| no| good| no| no|  ckd|  0.0|(24,[0,1,2,3,4,5,...|(24,[0,1,2,3,4,5,...|\n|  0|  0|  normal|  normal|notpresent|notpresent|yes| no| no| poor| no| no|  ckd|  0.0|(24,[0,1,2,3,4,5,...|(24,[0,1,2,3,4,5,...|\n|  0|  3|  normal|  normal|notpresent|notpresent|yes|yes|yes| good| no| no|  ckd|  0.0|[60.0,100.0,1.025...|[0.00497899486541...|\n|  1|  0|  normal|abnormal|   present|notpresent|yes| no|yes| good| no| no|  ckd|  0.0|[62.0,60.0,1.015,...|[0.01078866724045...|\n+---+---+--------+--------+----------+----------+---+---+---+-----+---+---+-----+-----+--------------------+--------------------+\nonly showing top 20 rows\n\n"
                }
            ], 
            "execution_count": 48
        }, 
        {
            "source": "for col in cat_col_list:\n      df_combine = df_combine.drop(col)\n        \ndf_combine.show()    ", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "+-----+-----+--------------------+--------------------+\n|class|label|            features|       features_norm|\n+-----+-----+--------------------+--------------------+\n|  ckd|  0.0|[48.0,80.0,1.02,1...|[0.00578524031092...|\n|  ckd|  0.0|(24,[0,1,2,3,4,5,...|(24,[0,1,2,3,4,5,...|\n|  ckd|  0.0|[62.0,80.0,1.01,4...|[0.00745615889881...|\n|  ckd|  0.0|[48.0,70.0,1.005,...|[0.00669791897052...|\n|  ckd|  0.0|(24,[0,1,2,3,4,5,...|(24,[0,1,2,3,4,5,...|\n|  ckd|  0.0|[60.0,90.0,1.015,...|[0.00726575654992...|\n|  ckd|  0.0|(24,[0,1,2,3,4,5,...|(24,[0,1,2,3,4,5,...|\n|  ckd|  0.0|[24.0,76.46907216...|[0.00313514838152...|\n|  ckd|  0.0|[52.0,100.0,1.015...|[0.00512271311618...|\n|  ckd|  0.0|[53.0,90.0,1.02,7...|[0.00420731087740...|\n|  ckd|  0.0|[50.0,60.0,1.01,4...|[0.00539933720326...|\n|  ckd|  0.0|[63.0,70.0,1.01,3...|[0.01195784007243...|\n|  ckd|  0.0|[68.0,70.0,1.015,...|[0.00530461973872...|\n|  ckd|  0.0|[68.0,70.0,1.0174...|[0.00761432819416...|\n|  ckd|  0.0|[68.0,80.0,1.01,1...|[0.00587589250918...|\n|  ckd|  0.0|[40.0,80.0,1.015,...|[0.00918713387836...|\n|  ckd|  0.0|(24,[0,1,2,3,4,5,...|(24,[0,1,2,3,4,5,...|\n|  ckd|  0.0|(24,[0,1,2,3,4,5,...|(24,[0,1,2,3,4,5,...|\n|  ckd|  0.0|[60.0,100.0,1.025...|[0.00497899486541...|\n|  ckd|  0.0|[62.0,60.0,1.015,...|[0.01078866724045...|\n+-----+-----+--------------------+--------------------+\nonly showing top 20 rows\n\n"
                }
            ], 
            "execution_count": 49
        }, 
        {
            "source": "df_combine = df_combine.drop(\"features\").drop(\"class\")\ndf_combine.show()", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "+-----+--------------------+\n|label|       features_norm|\n+-----+--------------------+\n|  0.0|[0.00578524031092...|\n|  0.0|(24,[0,1,2,3,4,5,...|\n|  0.0|[0.00745615889881...|\n|  0.0|[0.00669791897052...|\n|  0.0|(24,[0,1,2,3,4,5,...|\n|  0.0|[0.00726575654992...|\n|  0.0|(24,[0,1,2,3,4,5,...|\n|  0.0|[0.00313514838152...|\n|  0.0|[0.00512271311618...|\n|  0.0|[0.00420731087740...|\n|  0.0|[0.00539933720326...|\n|  0.0|[0.01195784007243...|\n|  0.0|[0.00530461973872...|\n|  0.0|[0.00761432819416...|\n|  0.0|[0.00587589250918...|\n|  0.0|[0.00918713387836...|\n|  0.0|(24,[0,1,2,3,4,5,...|\n|  0.0|(24,[0,1,2,3,4,5,...|\n|  0.0|[0.00497899486541...|\n|  0.0|[0.01078866724045...|\n+-----+--------------------+\nonly showing top 20 rows\n\n"
                }
            ], 
            "execution_count": 50
        }, 
        {
            "source": "#Split the dataset into training and testing dataset\n\nsplits = df_combine.randomSplit([0.8, 0.2])\ndf_train = splits[0]\ndf_test = splits[1]", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 51
        }, 
        {
            "source": "from pyspark.ml.classification import RandomForestClassifier\n\n# Train a RandomForest model.\nrf = RandomForestClassifier(labelCol=\"label\", featuresCol=\"features_norm\", numTrees=10)", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 52
        }, 
        {
            "source": "from pyspark.ml import Pipeline\npipeline = Pipeline(stages=[rf])", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 53
        }, 
        {
            "source": "model = pipeline.fit(df_train)", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 54
        }, 
        {
            "source": "prediction = model.transform(df_train)", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": 55
        }, 
        {
            "source": "\nprediction.printSchema()", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "root\n |-- label: double (nullable = false)\n |-- features_norm: vector (nullable = true)\n |-- rawPrediction: vector (nullable = true)\n |-- probability: vector (nullable = true)\n |-- prediction: double (nullable = false)\n\n"
                }
            ], 
            "execution_count": 56
        }, 
        {
            "source": "prediction.show()", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "+-----+--------------------+--------------------+--------------------+----------+\n|label|       features_norm|       rawPrediction|         probability|prediction|\n+-----+--------------------+--------------------+--------------------+----------+\n|  0.0|(24,[0,1,2,3,4,5,...|[5.33808840282524...|[0.53380884028252...|       0.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[8.81742424242424...|[0.88174242424242...|       0.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[6.29953758362849...|[0.62995375836284...|       0.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[8.53005706414797...|[0.85300570641479...|       0.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[1.97685648461357...|[0.19768564846135...|       1.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[7.58417065390749...|[0.75841706539074...|       0.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[6.65663097136781...|[0.66566309713678...|       0.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[9.08464696223316...|[0.90846469622331...|       0.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[9.61742424242424...|[0.96174242424242...|       0.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[9.78342041100661...|[0.97834204110066...|       0.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[9.70833333333333...|[0.97083333333333...|       0.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[9.58266283524904...|[0.95826628352490...|       0.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[6.80562087035771...|[0.68056208703577...|       0.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[7.62736636404348...|[0.76273663640434...|       0.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[9.51397596656217...|[0.95139759665621...|       0.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[5.48882899397253...|[0.54888289939725...|       0.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[6.25808840282524...|[0.62580884028252...|       0.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[6.62533227006911...|[0.66253322700691...|       0.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[5.57762813145588...|[0.55776281314558...|       0.0|\n|  0.0|(24,[0,1,2,3,4,5,...|[9.35564263322884...|[0.93556426332288...|       0.0|\n+-----+--------------------+--------------------+--------------------+----------+\nonly showing top 20 rows\n\n"
                }
            ], 
            "execution_count": 57
        }, 
        {
            "source": "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\nbinEval = MulticlassClassificationEvaluator().setMetricName(\"accuracy\") .setPredictionCol(\"prediction\").setLabelCol(\"label\")\n    \nbinEval.evaluate(prediction)", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 58, 
                    "output_type": "execute_result", 
                    "data": {
                        "text/plain": "0.99079754601227"
                    }, 
                    "metadata": {}
                }
            ], 
            "execution_count": 58
        }, 
        {
            "source": "#test on testing data\nprediction = model.transform(df_test)\n\nfrom pyspark.ml.evaluation import MulticlassClassificationEvaluator\nbinEval = MulticlassClassificationEvaluator().setMetricName(\"accuracy\") .setPredictionCol(\"prediction\").setLabelCol(\"label\")\n    \nbinEval.evaluate(prediction)", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 59, 
                    "output_type": "execute_result", 
                    "data": {
                        "text/plain": "0.9324324324324325"
                    }, 
                    "metadata": {}
                }
            ], 
            "execution_count": 59
        }, 
        {
            "source": "", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }, 
        {
            "source": "", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }, 
        {
            "source": "", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }, 
        {
            "source": "", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }, 
        {
            "source": "", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }, 
        {
            "source": "", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }, 
        {
            "source": "", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }, 
        {
            "source": "", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }, 
        {
            "source": "", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }, 
        {
            "source": "", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }, 
        {
            "source": "", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }, 
        {
            "source": "", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }, 
        {
            "source": "", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }, 
        {
            "source": "", 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "execution_count": null
        }
    ], 
    "metadata": {
        "kernelspec": {
            "display_name": "Python 3.5 with Spark", 
            "name": "python3", 
            "language": "python3"
        }, 
        "language_info": {
            "mimetype": "text/x-python", 
            "nbconvert_exporter": "python", 
            "version": "3.5.4", 
            "name": "python", 
            "pygments_lexer": "ipython3", 
            "file_extension": ".py", 
            "codemirror_mode": {
                "version": 3, 
                "name": "ipython"
            }
        }
    }, 
    "nbformat": 4
}