sagemaker-notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "6e2ab31c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "27615765",
   "metadata": {},
   "outputs": [],
   "source": [
    "client_s3 = boto3.client(\"s3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b4d40765",
   "metadata": {},
   "outputs": [],
   "source": [
    "s3_objects = client_s3.list_objects(Bucket=\"ingesting-data-super-heroes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "834322a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['csv_file/', 'csv_file/df_info.csv']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[i[\"Key\"] for i in s3_objects[\"Contents\"] if i[\"Key\"].lower().find(\"csv_file\")>=0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "228b12fa",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "client_s3.download_file(Bucket=\"ingesting-data-super-heroes\", \n",
    "                        Filename=\"df_info.csv\", Key=\"csv_file/df_info.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38fbe402",
   "metadata": {},
   "source": [
    "___"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "b6ad2c18",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"df_info.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "b150507d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>gender</th>\n",
       "      <th>eye_color</th>\n",
       "      <th>race</th>\n",
       "      <th>hair_color</th>\n",
       "      <th>height</th>\n",
       "      <th>publisher</th>\n",
       "      <th>skin_color</th>\n",
       "      <th>alignment</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A-Bomb</td>\n",
       "      <td>Male</td>\n",
       "      <td>yellow</td>\n",
       "      <td>Human</td>\n",
       "      <td>No Hair</td>\n",
       "      <td>203.0</td>\n",
       "      <td>Marvel Comics</td>\n",
       "      <td>-</td>\n",
       "      <td>good</td>\n",
       "      <td>441.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Abe Sapien</td>\n",
       "      <td>Male</td>\n",
       "      <td>blue</td>\n",
       "      <td>Icthyo Sapien</td>\n",
       "      <td>No Hair</td>\n",
       "      <td>191.0</td>\n",
       "      <td>Dark Horse Comics</td>\n",
       "      <td>blue</td>\n",
       "      <td>good</td>\n",
       "      <td>65.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Abin Sur</td>\n",
       "      <td>Male</td>\n",
       "      <td>blue</td>\n",
       "      <td>Ungaran</td>\n",
       "      <td>No Hair</td>\n",
       "      <td>185.0</td>\n",
       "      <td>DC Comics</td>\n",
       "      <td>red</td>\n",
       "      <td>good</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Abomination</td>\n",
       "      <td>Male</td>\n",
       "      <td>green</td>\n",
       "      <td>Human / Radiation</td>\n",
       "      <td>No Hair</td>\n",
       "      <td>203.0</td>\n",
       "      <td>Marvel Comics</td>\n",
       "      <td>-</td>\n",
       "      <td>bad</td>\n",
       "      <td>441.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Abraxas</td>\n",
       "      <td>Male</td>\n",
       "      <td>blue</td>\n",
       "      <td>Cosmic Entity</td>\n",
       "      <td>Black</td>\n",
       "      <td>-99.0</td>\n",
       "      <td>Marvel Comics</td>\n",
       "      <td>-</td>\n",
       "      <td>bad</td>\n",
       "      <td>-99.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          name gender eye_color               race hair_color  height  \\\n",
       "0       A-Bomb   Male    yellow              Human    No Hair   203.0   \n",
       "1   Abe Sapien   Male      blue      Icthyo Sapien    No Hair   191.0   \n",
       "2     Abin Sur   Male      blue            Ungaran    No Hair   185.0   \n",
       "3  Abomination   Male     green  Human / Radiation    No Hair   203.0   \n",
       "4      Abraxas   Male      blue      Cosmic Entity      Black   -99.0   \n",
       "\n",
       "           publisher skin_color alignment  weight  \n",
       "0      Marvel Comics          -      good   441.0  \n",
       "1  Dark Horse Comics       blue      good    65.0  \n",
       "2          DC Comics        red      good    90.0  \n",
       "3      Marvel Comics          -       bad   441.0  \n",
       "4      Marvel Comics          -       bad   -99.0  "
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "ef325d96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(734, 10)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "0d2fa79c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop([\"skin_color\",\"name\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "7cfdb16e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[(df.weight>=0) & (df.height>=0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "1df39c9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Turn the multiclass classification problem to binary classification problem selecting only two label.\n",
    "\n",
    "df = df[df.publisher.isin([\"Marvel Comics\", \"DC Comics\"])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "a30922ea",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gender</th>\n",
       "      <th>eye_color</th>\n",
       "      <th>race</th>\n",
       "      <th>hair_color</th>\n",
       "      <th>height</th>\n",
       "      <th>publisher</th>\n",
       "      <th>alignment</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Male</td>\n",
       "      <td>yellow</td>\n",
       "      <td>Human</td>\n",
       "      <td>No Hair</td>\n",
       "      <td>203.0</td>\n",
       "      <td>Marvel Comics</td>\n",
       "      <td>good</td>\n",
       "      <td>441.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Male</td>\n",
       "      <td>blue</td>\n",
       "      <td>Ungaran</td>\n",
       "      <td>No Hair</td>\n",
       "      <td>185.0</td>\n",
       "      <td>DC Comics</td>\n",
       "      <td>good</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Male</td>\n",
       "      <td>green</td>\n",
       "      <td>Human / Radiation</td>\n",
       "      <td>No Hair</td>\n",
       "      <td>203.0</td>\n",
       "      <td>Marvel Comics</td>\n",
       "      <td>bad</td>\n",
       "      <td>441.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Male</td>\n",
       "      <td>blue</td>\n",
       "      <td>Human</td>\n",
       "      <td>No Hair</td>\n",
       "      <td>193.0</td>\n",
       "      <td>Marvel Comics</td>\n",
       "      <td>bad</td>\n",
       "      <td>122.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Male</td>\n",
       "      <td>blue</td>\n",
       "      <td>Human</td>\n",
       "      <td>Blond</td>\n",
       "      <td>185.0</td>\n",
       "      <td>DC Comics</td>\n",
       "      <td>good</td>\n",
       "      <td>88.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  gender eye_color               race hair_color  height      publisher  \\\n",
       "0   Male    yellow              Human    No Hair   203.0  Marvel Comics   \n",
       "2   Male      blue            Ungaran    No Hair   185.0      DC Comics   \n",
       "3   Male     green  Human / Radiation    No Hair   203.0  Marvel Comics   \n",
       "5   Male      blue              Human    No Hair   193.0  Marvel Comics   \n",
       "7   Male      blue              Human      Blond   185.0      DC Comics   \n",
       "\n",
       "  alignment  weight  \n",
       "0      good   441.0  \n",
       "2      good    90.0  \n",
       "3       bad   441.0  \n",
       "5       bad   122.0  \n",
       "7      good    88.0  "
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db3d10c9",
   "metadata": {},
   "source": [
    "Label encoding."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "c11c416d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_encoder = LabelEncoder()\n",
    "\n",
    "df_encoded = pd.DataFrame({})\n",
    "for col in df.drop(\"publisher\", axis=1).columns:\n",
    "    if col not in [\"height\", \"weight\"]:\n",
    "        df_encoded[col] = label_encoder.fit_transform(df[col])\n",
    "    else:\n",
    "        df_encoded[col] = df[col].values\n",
    "df_encoded[\"publisher\"] = df[\"publisher\"].values\n",
    "\n",
    "# No missing values.\n",
    "df_encoded.isnull().any().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b61c1c9c",
   "metadata": {},
   "source": [
    "Balance the dataset. Split train, test and validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "41d62476",
   "metadata": {},
   "outputs": [],
   "source": [
    "# %90 train and %10 holdout dataset\n",
    "\n",
    "df_train, df_holdout = train_test_split(df_encoded, test_size=0.1, stratify=df_encoded[\"publisher\"])\n",
    "\n",
    "# Split holdout data into 50% validation and 50% test\n",
    "\n",
    "df_val, df_test = train_test_split(df_holdout, test_size=0.5, stratify=df_holdout[\"publisher\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "904edc4b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gender</th>\n",
       "      <th>eye_color</th>\n",
       "      <th>race</th>\n",
       "      <th>hair_color</th>\n",
       "      <th>height</th>\n",
       "      <th>alignment</th>\n",
       "      <th>weight</th>\n",
       "      <th>publisher</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>193.0</td>\n",
       "      <td>2</td>\n",
       "      <td>98.0</td>\n",
       "      <td>Marvel Comics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>201.0</td>\n",
       "      <td>2</td>\n",
       "      <td>216.0</td>\n",
       "      <td>Marvel Comics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>34</td>\n",
       "      <td>6</td>\n",
       "      <td>170.0</td>\n",
       "      <td>1</td>\n",
       "      <td>59.0</td>\n",
       "      <td>Marvel Comics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>185.0</td>\n",
       "      <td>2</td>\n",
       "      <td>146.0</td>\n",
       "      <td>DC Comics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>169</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>211.0</td>\n",
       "      <td>1</td>\n",
       "      <td>104.0</td>\n",
       "      <td>Marvel Comics</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     gender  eye_color  race  hair_color  height  alignment  weight  \\\n",
       "28        1         16     1          15   193.0          2    98.0   \n",
       "52        2          0     0          14   201.0          2   216.0   \n",
       "355       1          3    34           6   170.0          1    59.0   \n",
       "24        2          3     6           4   185.0          2   146.0   \n",
       "169       1          5     0           2   211.0          1   104.0   \n",
       "\n",
       "         publisher  \n",
       "28   Marvel Comics  \n",
       "52   Marvel Comics  \n",
       "355  Marvel Comics  \n",
       "24       DC Comics  \n",
       "169  Marvel Comics  "
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_val.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfe69cae",
   "metadata": {},
   "source": [
    "SageMaker Processing Jobs, let us parallelize the processing over many nodes in a cluster. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "f3c8cd96",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.sklearn.processing import SKLearnProcessor\n",
    "from sagemaker.processing import ProcessingInput, ProcessingOutput\n",
    "from sagemaker import get_execution_role"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "83394042",
   "metadata": {},
   "outputs": [],
   "source": [
    "role = get_execution_role()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "df3a2bef",
   "metadata": {},
   "outputs": [],
   "source": [
    "processor = SKLearnProcessor(framework_version='0.20.0',\n",
    "                             role=role,\n",
    "                             instance_type='ml.m5.xlarge',\n",
    "                             instance_count=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f1950dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.processing import ProcessingInput, ProcessingOutput\n",
    "\n",
    "input_data = \"s3://ingesting-data-super-heroes/csv_file/df_info.csv\"\n",
    "\n",
    "processor.run(\n",
    "    code=\"preprocessing.py\",\n",
    "    inputs=[ProcessingInput(source=input_data, destination=\"/opt/ml/processing/input\")],\n",
    "    outputs=[\n",
    "        ProcessingOutput(output_name=\"train_data\", source=\"/opt/ml/processing/train\"),\n",
    "        ProcessingOutput(output_name=\"test_data\", source=\"/opt/ml/processing/test\"),\n",
    "        ProcessingOutput(output_name=\"val_data\", source=\"/opt/ml/processing/val\"),\n",
    "    ],\n",
    ")\n",
    "\n",
    "preprocessing_job_description = processor.jobs[-1].describe()\n",
    "\n",
    "output_config = preprocessing_job_description[\"ProcessingOutputConfig\"]\n",
    "for output in output_config[\"Outputs\"]:\n",
    "    if output[\"OutputName\"] == \"train_data\":\n",
    "        preprocessed_training_data = output[\"S3Output\"][\"S3Uri\"]\n",
    "    if output[\"OutputName\"] == \"test_data\":\n",
    "        preprocessed_test_data = output[\"S3Output\"][\"S3Uri\"]\n",
    "    if output[\"OutputName\"] == \"val_data\":\n",
    "        preprocessed_val_data = output[\"S3Output\"][\"S3Uri\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a07becc5",
   "metadata": {},
   "source": [
    "Get preprocessed data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "319a8d93",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(preprocessed_training_data+\"/df_train.csv\")\n",
    "df_test = pd.read_csv(preprocessed_test_data+\"/df_test.csv\")\n",
    "df_val = pd.read_csv(preprocessed_val_data+\"/df_val.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50dc6aae",
   "metadata": {},
   "source": [
    "___"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}