training/training_sagemaker_resnet50_mercedes-models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "from sagemaker.pytorch import PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define input image dimensions for resizing\n",
    "height = 448\n",
    "width = 448\n",
    "\n",
    "# Define model hyperparameters\n",
    "lr = 0.01\n",
    "momentum = 0.9\n",
    "T_0 = 225 # e.g. 899 / 4 (train_dataset_size / batch_size)\n",
    "T_mult = 1\n",
    "epochs = 20\n",
    "batch_size = 4 # For both train and test sets\n",
    "\n",
    "# Define number of layers for the ResNet neural network, select from [18, 34, 50 ,101, 152]\n",
    "num_layers = 50\n",
    "\n",
    "pretrained_weights = True\n",
    "unfreeze_all_layers = False # i.e. Default: 'False', unfreezes last layer only for tuning\n",
    "\n",
    "train_augmentation = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model = './model/resnet_sgd_cosineannealing_inference.py'\n",
    "model = './model/resnet_sgd_cosineannealing_inference.py'\n",
    "\n",
    "model_save_name = 'ResNet{}-{}x{}-variants-5'.format(num_layers, height, width)\n",
    "\n",
    "model_save_path = 's3://car-tagging-datasets/sagemaker-models/trained-model-output'\n",
    "model_code_save_path = 's3://car-tagging-datasets/sagemaker-models/trained-model-custom-code'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Warm Restart Parameters:\n",
    "\n",
    "bucket = None\n",
    "saved_model_path = None\n",
    "\n",
    "# Uncomment (if necessary) and amend the below paths for warm restarts using custom pre-trained weights\n",
    "\n",
    "# bucket = 'custom-labels-console-eu-west-1-52749662e9'\n",
    "# saved_model_id = 'resnet50-sgd-cosineannealing-448x448-2021-01-08-14-07-00-745'\n",
    "# saved_model_path = 'for-custom-labels/train-dataset-augmented/sagemaker-pytorch-models/{}/output/model.tar.gz'.format(saved_model_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = 's3://car-tagging-datasets/all-variants/variants-5/train/'\n",
    "\n",
    "test_complex = 's3://car-tagging-datasets/all-variants/variants-5/test-complex/'\n",
    "\n",
    "test_showroom = 's3://car-tagging-datasets/all-variants/variants-5/test/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "sagemaker_session = sagemaker.Session()\n",
    "\n",
    "role = sagemaker.get_execution_role()\n",
    "\n",
    "#Check the status of dataloader\n",
    "estimator = PyTorch(entry_point=model,\n",
    "                    role=role,\n",
    "                    framework_version='1.4.0',\n",
    "                    py_version='py3',\n",
    "                    instance_count=1,\n",
    "                    instance_type='ml.g4dn.xlarge',\n",
    "                    base_job_name=model_save_name,\n",
    "                    output_path=model_save_path,\n",
    "#                     code_location=custom_code_uri,\n",
    "                    metric_definitions=[\n",
    "                        {'Name': 'train:loss', 'Regex': 'train Loss: (.*?);'},\n",
    "                        {'Name': 'train:acc', 'Regex': 'train Acc: (.*?);'},\n",
    "                        {'Name': 'validation:loss', 'Regex': 'validation Loss: (.*?);'},\n",
    "                        {'Name': 'validation:acc', 'Regex': 'validation Acc: (.*?);'},\n",
    "                        {'Name': 'learning_rate', 'Regex': 'lr: (.*?);'},\n",
    "                        {'Name': 'epoch', 'Regex': 'epoch: (.*?);'},\n",
    "                        {'Name': 'train_f1_score', 'Regex': 'train Avg. F1 Score: (.*?);'},\n",
    "                        {'Name': 'validation_f1_score', 'Regex': 'validation Avg. F1 Score: (.*?);'},\n",
    "                        {'Name': 'test_f1_score', 'Regex': 'test Avg. F1 Score: (.*?);'},\n",
    "                        {'Name': 'classification_report', 'Regex': 'classification_report: (.*?);'}\n",
    "                    ],\n",
    "                    hyperparameters={\n",
    "                        'image-height': height,\n",
    "                        'image-width': width,\n",
    "                        'batch-size': batch_size,\n",
    "                        'epochs': epochs,\n",
    "                        'lr': lr,\n",
    "                        'momentum': momentum,\n",
    "                        'T_0': T_0,\n",
    "                        'T_mult': T_mult,\n",
    "                        'num-layers': num_layers,\n",
    "                        'pretrained-weights': pretrained_weights,\n",
    "                        's3-bucket': bucket,\n",
    "                        'warm-restart': saved_model_path,\n",
    "                        'unfreeze-all-layers': unfreeze_all_layers,\n",
    "                        'train-augmentation': train_augmentation\n",
    "                    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-06-02 09:49:08 Starting - Starting the training job...\n",
      "2021-06-02 09:49:30 Starting - Launching requested ML instancesProfilerReport-1622627347: InProgress\n",
      "......\n",
      "2021-06-02 09:50:30 Starting - Preparing the instances for training......\n",
      "2021-06-02 09:51:31 Downloading - Downloading input data...\n",
      "2021-06-02 09:51:52 Training - Downloading the training image......\n",
      "2021-06-02 09:53:05 Training - Training image download completed. Training in progress..\u001b[34mbash: cannot set terminal process group (-1): Inappropriate ioctl for device\u001b[0m\n",
      "\u001b[34mbash: no job control in this shell\u001b[0m\n",
      "\u001b[34m2021-06-02 09:53:07,115 sagemaker-containers INFO     Imported framework sagemaker_pytorch_container.training\u001b[0m\n",
      "\u001b[34m2021-06-02 09:53:07,136 sagemaker_pytorch_container.training INFO     Block until all host DNS lookups succeed.\u001b[0m\n",
      "\u001b[34m2021-06-02 09:53:07,139 sagemaker_pytorch_container.training INFO     Invoking user training script.\u001b[0m\n",
      "\u001b[34m2021-06-02 09:53:07,450 sagemaker-containers INFO     Module default_user_module_name does not provide a setup.py. \u001b[0m\n",
      "\u001b[34mGenerating setup.py\u001b[0m\n",
      "\u001b[34m2021-06-02 09:53:07,450 sagemaker-containers INFO     Generating setup.cfg\u001b[0m\n",
      "\u001b[34m2021-06-02 09:53:07,451 sagemaker-containers INFO     Generating MANIFEST.in\u001b[0m\n",
      "\u001b[34m2021-06-02 09:53:07,451 sagemaker-containers INFO     Installing module with the following command:\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/python3.6 -m pip install . \u001b[0m\n",
      "\u001b[34mProcessing /tmp/tmpkd7cx0su/module_dir\u001b[0m\n",
      "\u001b[34mBuilding wheels for collected packages: default-user-module-name\n",
      "  Building wheel for default-user-module-name (setup.py): started\u001b[0m\n",
      "\u001b[34m  Building wheel for default-user-module-name (setup.py): finished with status 'done'\n",
      "  Created wheel for default-user-module-name: filename=default_user_module_name-1.0.0-py2.py3-none-any.whl size=11205 sha256=bfe1552c0c39a15409bb8e498cb805e9e2c151efb7ed5b4b5fb531b34bde8068\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-vlrsvy3w/wheels/9c/95/09/32a6a1b1fb006800d523cb6ae03bb6f06c1dbbea432471b92c\u001b[0m\n",
      "\u001b[34mSuccessfully built default-user-module-name\u001b[0m\n",
      "\u001b[34mInstalling collected packages: default-user-module-name\u001b[0m\n",
      "\u001b[34mSuccessfully installed default-user-module-name-1.0.0\u001b[0m\n",
      "\u001b[34m2021-06-02 09:53:09,838 sagemaker-containers INFO     Invoking user script\n",
      "\u001b[0m\n",
      "\u001b[34mTraining Env:\n",
      "\u001b[0m\n",
      "\u001b[34m{\n",
      "    \"additional_framework_parameters\": {},\n",
      "    \"channel_input_dirs\": {\n",
      "        \"test\": \"/opt/ml/input/data/test\",\n",
      "        \"validation\": \"/opt/ml/input/data/validation\",\n",
      "        \"train\": \"/opt/ml/input/data/train\"\n",
      "    },\n",
      "    \"current_host\": \"algo-1\",\n",
      "    \"framework_module\": \"sagemaker_pytorch_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"hyperparameters\": {\n",
      "        \"num-layers\": 50,\n",
      "        \"image-width\": 448,\n",
      "        \"T_0\": 225,\n",
      "        \"lr\": 0.01,\n",
      "        \"train-augmentation\": false,\n",
      "        \"unfreeze-all-layers\": false,\n",
      "        \"pretrained-weights\": true,\n",
      "        \"momentum\": 0.9,\n",
      "        \"batch-size\": 4,\n",
      "        \"warm-restart\": null,\n",
      "        \"image-height\": 448,\n",
      "        \"s3-bucket\": null,\n",
      "        \"epochs\": 20,\n",
      "        \"T_mult\": 1\n",
      "    },\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {\n",
      "        \"test\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        },\n",
      "        \"validation\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        },\n",
      "        \"train\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        }\n",
      "    },\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"is_master\": true,\n",
      "    \"job_name\": \"ResNet50-448x448-orientation-view-phase-2021-06-02-09-49-07-532\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://car-tagging-datasets/ResNet50-448x448-orientation-view-phase-2021-06-02-09-49-07-532/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"resnet_sgd_cosineannealing_inference\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 4,\n",
      "    \"num_gpus\": 1,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-1\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\"\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"resnet_sgd_cosineannealing_inference.py\"\u001b[0m\n",
      "\u001b[34m}\n",
      "\u001b[0m\n",
      "\u001b[34mEnvironment variables:\n",
      "\u001b[0m\n",
      "\u001b[34mSM_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[34mSM_HPS={\"T_0\":225,\"T_mult\":1,\"batch-size\":4,\"epochs\":20,\"image-height\":448,\"image-width\":448,\"lr\":0.01,\"momentum\":0.9,\"num-layers\":50,\"pretrained-weights\":true,\"s3-bucket\":null,\"train-augmentation\":false,\"unfreeze-all-layers\":false,\"warm-restart\":null}\u001b[0m\n",
      "\u001b[34mSM_USER_ENTRY_POINT=resnet_sgd_cosineannealing_inference.py\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_PARAMS={}\u001b[0m\n",
      "\u001b[34mSM_RESOURCE_CONFIG={\"current_host\":\"algo-1\",\"hosts\":[\"algo-1\"],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[34mSM_INPUT_DATA_CONFIG={\"test\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"train\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"validation\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}}\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[34mSM_CHANNELS=[\"test\",\"train\",\"validation\"]\u001b[0m\n",
      "\u001b[34mSM_CURRENT_HOST=algo-1\u001b[0m\n",
      "\u001b[34mSM_MODULE_NAME=resnet_sgd_cosineannealing_inference\u001b[0m\n",
      "\u001b[34mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_MODULE=sagemaker_pytorch_container.training:main\u001b[0m\n",
      "\u001b[34mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[34mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[34mSM_NUM_CPUS=4\u001b[0m\n",
      "\u001b[34mSM_NUM_GPUS=1\u001b[0m\n",
      "\u001b[34mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[34mSM_MODULE_DIR=s3://car-tagging-datasets/ResNet50-448x448-orientation-view-phase-2021-06-02-09-49-07-532/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[34mSM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{\"test\":\"/opt/ml/input/data/test\",\"train\":\"/opt/ml/input/data/train\",\"validation\":\"/opt/ml/input/data/validation\"},\"current_host\":\"algo-1\",\"framework_module\":\"sagemaker_pytorch_container.training:main\",\"hosts\":[\"algo-1\"],\"hyperparameters\":{\"T_0\":225,\"T_mult\":1,\"batch-size\":4,\"epochs\":20,\"image-height\":448,\"image-width\":448,\"lr\":0.01,\"momentum\":0.9,\"num-layers\":50,\"pretrained-weights\":true,\"s3-bucket\":null,\"train-augmentation\":false,\"unfreeze-all-layers\":false,\"warm-restart\":null},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{\"test\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"train\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"validation\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}},\"input_dir\":\"/opt/ml/input\",\"is_master\":true,\"job_name\":\"ResNet50-448x448-orientation-view-phase-2021-06-02-09-49-07-532\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://car-tagging-datasets/ResNet50-448x448-orientation-view-phase-2021-06-02-09-49-07-532/source/sourcedir.tar.gz\",\"module_name\":\"resnet_sgd_cosineannealing_inference\",\"network_interface_name\":\"eth0\",\"num_cpus\":4,\"num_gpus\":1,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_host\":\"algo-1\",\"hosts\":[\"algo-1\"],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"resnet_sgd_cosineannealing_inference.py\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ARGS=[\"--T_0\",\"225\",\"--T_mult\",\"1\",\"--batch-size\",\"4\",\"--epochs\",\"20\",\"--image-height\",\"448\",\"--image-width\",\"448\",\"--lr\",\"0.01\",\"--momentum\",\"0.9\",\"--num-layers\",\"50\",\"--pretrained-weights\",\"True\",\"--s3-bucket\",\"\",\"--train-augmentation\",\"False\",\"--unfreeze-all-layers\",\"False\",\"--warm-restart\",\"\"]\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_TEST=/opt/ml/input/data/test\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_VALIDATION=/opt/ml/input/data/validation\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_TRAIN=/opt/ml/input/data/train\u001b[0m\n",
      "\u001b[34mSM_HP_NUM-LAYERS=50\u001b[0m\n",
      "\u001b[34mSM_HP_IMAGE-WIDTH=448\u001b[0m\n",
      "\u001b[34mSM_HP_T_0=225\u001b[0m\n",
      "\u001b[34mSM_HP_LR=0.01\u001b[0m\n",
      "\u001b[34mSM_HP_TRAIN-AUGMENTATION=false\u001b[0m\n",
      "\u001b[34mSM_HP_UNFREEZE-ALL-LAYERS=false\u001b[0m\n",
      "\u001b[34mSM_HP_PRETRAINED-WEIGHTS=true\u001b[0m\n",
      "\u001b[34mSM_HP_MOMENTUM=0.9\u001b[0m\n",
      "\u001b[34mSM_HP_BATCH-SIZE=4\u001b[0m\n",
      "\u001b[34mSM_HP_WARM-RESTART=\u001b[0m\n",
      "\u001b[34mSM_HP_IMAGE-HEIGHT=448\u001b[0m\n",
      "\u001b[34mSM_HP_S3-BUCKET=\u001b[0m\n",
      "\u001b[34mSM_HP_EPOCHS=20\u001b[0m\n",
      "\u001b[34mSM_HP_T_MULT=1\u001b[0m\n",
      "\u001b[34mPYTHONPATH=/opt/ml/code:/opt/conda/bin:/opt/conda/lib/python36.zip:/opt/conda/lib/python3.6:/opt/conda/lib/python3.6/lib-dynload:/opt/conda/lib/python3.6/site-packages\n",
      "\u001b[0m\n",
      "\u001b[34mInvoking script with the following command:\n",
      "\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/python3.6 resnet_sgd_cosineannealing_inference.py --T_0 225 --T_mult 1 --batch-size 4 --epochs 20 --image-height 448 --image-width 448 --lr 0.01 --momentum 0.9 --num-layers 50 --pretrained-weights True --s3-bucket  --train-augmentation False --unfreeze-all-layers False --warm-restart \n",
      "\n",
      "\u001b[0m\n",
      "\u001b[34mGPU NAME: Tesla T4\u001b[0m\n",
      "\u001b[34mEpoch 1/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34m[2021-06-02 09:53:16.728 algo-1:43 INFO json_config.py:90] Creating hook from json_config at /opt/ml/input/config/debughookconfig.json.\u001b[0m\n",
      "\u001b[34m[2021-06-02 09:53:16.728 algo-1:43 INFO hook.py:192] tensorboard_dir has not been set for the hook. SMDebug will not be exporting tensorboard summaries.\u001b[0m\n",
      "\u001b[34m[2021-06-02 09:53:16.728 algo-1:43 INFO hook.py:237] Saving to /opt/ml/output/tensors\u001b[0m\n",
      "\u001b[34m[2021-06-02 09:53:16.728 algo-1:43 INFO state_store.py:67] The checkpoint config file /opt/ml/input/config/checkpointconfig.json does not exist.\u001b[0m\n",
      "\u001b[34m[2021-06-02 09:53:16.741 algo-1:43 INFO hook.py:382] Monitoring the collections: losses\u001b[0m\n",
      "\u001b[34m[2021-06-02 09:53:16.741 algo-1:43 INFO hook.py:443] Hook is writing from the hook with pid: 43\n",
      "\u001b[0m\n",
      "\u001b[34mtrain Loss: 1.2809; train Acc: 0.7825;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 2.2203; validation Acc: 0.6149;\u001b[0m\n",
      "\u001b[34mepoch: 1; lr: 0.0009963431452563332;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 2/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.9976; train Acc: 0.8476;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 1.7451; validation Acc: 0.7143;\u001b[0m\n",
      "\u001b[34mepoch: 2; lr: 0.003588292715785618;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 3/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.6349; train Acc: 0.8866;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 1.6352; validation Acc: 0.6894;\u001b[0m\n",
      "\u001b[34mepoch: 3; lr: 0.006742860236609076;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 4/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.5087; train Acc: 0.9126;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 4.2429; validation Acc: 0.5528;\u001b[0m\n",
      "\u001b[34mepoch: 4; lr: 0.009202833017478422;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 5/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.5287; train Acc: 0.9033;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 1.3717; validation Acc: 0.7640;\u001b[0m\n",
      "\u001b[34mepoch: 5; lr: 1.2179748700878457e-05;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 6/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.7134; train Acc: 0.9040;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 1.6285; validation Acc: 0.7391;\u001b[0m\n",
      "\u001b[34mepoch: 6; lr: 0.0012150247217412185;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 7/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.3913; train Acc: 0.9139;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 1.6647; validation Acc: 0.7453;\u001b[0m\n",
      "\u001b[34mepoch: 7; lr: 0.003926323364164684;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 8/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.5669; train Acc: 0.9238;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 3.9880; validation Acc: 0.6460;\u001b[0m\n",
      "\u001b[34mepoch: 8; lr: 0.007065522149122709;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 9/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.4902; train Acc: 0.9157;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 3.1481; validation Acc: 0.6894;\u001b[0m\n",
      "\u001b[34mepoch: 9; lr: 0.009381533400219317;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 10/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.4037; train Acc: 0.9269;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 1.7107; validation Acc: 0.7888;\u001b[0m\n",
      "\u001b[34mepoch: 10; lr: 4.865965629214875e-05;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 11/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.6904; train Acc: 0.9077;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 1.8990; validation Acc: 0.7764;\u001b[0m\n",
      "\u001b[34mepoch: 11; lr: 0.0014521463173173949;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 12/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.4786; train Acc: 0.9275;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 1.8020; validation Acc: 0.7826;\u001b[0m\n",
      "\u001b[34mepoch: 12; lr: 0.004269584857187943;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 13/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.4213; train Acc: 0.9325;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 2.0286; validation Acc: 0.7578;\u001b[0m\n",
      "\u001b[34mepoch: 13; lr: 0.007378121045351377;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 14/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.4413; train Acc: 0.9306;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 3.3918; validation Acc: 0.6398;\u001b[0m\n",
      "\u001b[34mepoch: 14; lr: 0.009538887392664544;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 15/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.4462; train Acc: 0.9250;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 1.9293; validation Acc: 0.7826;\u001b[0m\n",
      "\u001b[34mepoch: 15; lr: 0.00010926199633097155;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 16/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.3786; train Acc: 0.9362;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 1.8126; validation Acc: 0.7640;\u001b[0m\n",
      "\u001b[34mepoch: 16; lr: 0.001706552699406599;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 17/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.3971; train Acc: 0.9306;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 2.5581; validation Acc: 0.6770;\u001b[0m\n",
      "\u001b[34mepoch: 17; lr: 0.0046164048593659085;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 18/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.2937; train Acc: 0.9418;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 3.0365; validation Acc: 0.7019;\u001b[0m\n",
      "\u001b[34mepoch: 18; lr: 0.007679133974894983;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 19/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.3427; train Acc: 0.9362;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 1.8662; validation Acc: 0.7764;\u001b[0m\n",
      "\u001b[34mepoch: 19; lr: 0.009674128381980071;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 20/20\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.4460; train Acc: 0.9337;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 2.0283; validation Acc: 0.7453;\u001b[0m\n",
      "\u001b[34mepoch: 20; lr: 0.00019369152030840553;\n",
      "\u001b[0m\n",
      "\u001b[34mTraining complete in 32m 16s\u001b[0m\n",
      "\u001b[34mBest validation Acc: 0.788820\u001b[0m\n",
      "\u001b[34m/opt/ml/model/model.pth\n",
      "\u001b[0m\n",
      "\u001b[34mEvaluating best weights:\u001b[0m\n",
      "\u001b[34m--------------------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.0607 Acc: 0.9857\u001b[0m\n",
      "\u001b[34mtrain Avg. F1 Score: 0.986;\u001b[0m\n",
      "\u001b[34mclassification_report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "        back       0.95      0.99      0.97       225\n",
      "       front       0.99      0.96      0.98       253\n",
      "        side       0.99      0.99      0.99      1136\n",
      "\n",
      "    accuracy                           0.99      1614\n",
      "   macro avg       0.98      0.98      0.98      1614\u001b[0m\n",
      "\u001b[34mweighted avg       0.99      0.99      0.99      1614\u001b[0m\n",
      "\u001b[34m;\n",
      "\u001b[0m\n",
      "\u001b[34mvalidation Loss: 1.7107 Acc: 0.7888\u001b[0m\n",
      "\u001b[34mvalidation Avg. F1 Score: 0.789;\u001b[0m\n",
      "\u001b[34mclassification_report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "        back       0.71      0.86      0.78        35\n",
      "       front       0.82      0.73      0.77        51\n",
      "        side       0.81      0.80      0.81        75\n",
      "\n",
      "    accuracy                           0.79       161\n",
      "   macro avg       0.78      0.79      0.79       161\u001b[0m\n",
      "\u001b[34mweighted avg       0.79      0.79      0.79       161\u001b[0m\n",
      "\u001b[34m;\n",
      "\u001b[0m\n",
      "\n",
      "2021-06-02 10:27:31 Uploading - Uploading generated training model\u001b[34mtest Loss: 2.9112 Acc: 0.7121\u001b[0m\n",
      "\u001b[34mtest Avg. F1 Score: 0.736;\u001b[0m\n",
      "\u001b[34mclassification_report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "        back       0.39      1.00      0.57        54\n",
      "       front       0.54      1.00      0.70        36\n",
      "        side       1.00      0.63      0.77       306\n",
      "\n",
      "    accuracy                           0.71       396\n",
      "   macro avg       0.64      0.88      0.68       396\u001b[0m\n",
      "\u001b[34mweighted avg       0.88      0.71      0.74       396\u001b[0m\n",
      "\u001b[34m;\n",
      "\u001b[0m\n",
      "\u001b[34m2021-06-02 10:27:28,448 sagemaker-containers INFO     Reporting training SUCCESS\u001b[0m\n",
      "\n",
      "2021-06-02 10:27:59 Completed - Training job completed\n",
      "ProfilerReport-1622627347: IssuesFound\n",
      "Training seconds: 2190\n",
      "Billable seconds: 2190\n"
     ]
    }
   ],
   "source": [
    "estimator.fit({'train': train,\n",
    "               'validation': test_complex,\n",
    "               'test': test_showroom})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}