training/training_sagemaker_resnet18_mercedes-vs-not.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Open set classification: transfer learning using same learning as the mercedes car-tagging model\n",
    "Differences:\n",
    "- smaller resnet (18)\n",
    "- retrained on mercedes vs. non-mercedes dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mercedes vs. non-mercedes dataset\n",
    "2 class dataset split into train,val,test. \n",
    "- Class 0 (mercedes) images taken from all Mercedes variants from s3://car-tagging-datasets/all-variants//variants-5. Test-complex images are augmented 6 times (using augmentation script) so that the showroom and complex images are balanced. A subset is taken for the final dataset.\n",
    "- Class 1 (non-mercedes) subset from http://ai.stanford.edu/~jkrause/cars/car_dataset.html "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "from sagemaker.pytorch import PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define input image dimensions for resizing\n",
    "height = 448\n",
    "width = 448\n",
    "\n",
    "# Define model hyperparameters\n",
    "lr = 0.01\n",
    "momentum = 0.9\n",
    "T_0 = 225 # e.g. 899 / 4 (train_dataset_size / batch_size)\n",
    "T_mult = 1\n",
    "epochs = 15\n",
    "batch_size = 4 # For both train and test sets\n",
    "\n",
    "# Define number of layers for the ResNet neural network, select from [18, 34, 50 ,101, 152]\n",
    "num_layers = 18\n",
    "\n",
    "pretrained_weights = True\n",
    "unfreeze_all_layers = False # i.e. Default: 'False', unfreezes last layer only for tuning\n",
    "\n",
    "train_augmentation = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model = './model/resnet_sgd_cosineannealing_inference.py'\n",
    "model = './model/resnet_sgd_cosineannealing_inference.py'\n",
    "\n",
    "\n",
    "model_save_name = 'ResNet-cars-open-set'\n",
    "model_save_path = 's3://custom-labels-console-eu-west-1-52749662e9/for-custom-labels/cars-open-set/sagemaker-pytorch-models/'\n",
    "#model_code_save_path = 's3://car-tagging-datasets/sagemaker-models/trained-model-custom-code'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Warm Restart Parameters:\n",
    "\n",
    "bucket = None\n",
    "saved_model_path = None\n",
    "\n",
    "# Uncomment (if necessary) and amend the below paths for warm restarts using custom pre-trained weights\n",
    "\n",
    "# bucket = 'custom-labels-console-eu-west-1-52749662e9'\n",
    "# saved_model_id = 'resnet50-sgd-cosineannealing-448x448-2021-01-08-14-07-00-745'\n",
    "# saved_model_path = 'for-custom-labels/train-dataset-augmented/sagemaker-pytorch-models/{}/output/model.tar.gz'.format(saved_model_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = 's3://car-tagging-datasets/open-set/train/'\n",
    "validation = 's3://car-tagging-datasets/open-set/validation/'\n",
    "test = 's3://car-tagging-datasets/open-set/test/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "sagemaker_session = sagemaker.Session()\n",
    "\n",
    "role = sagemaker.get_execution_role()\n",
    "\n",
    "#Check the status of dataloader\n",
    "estimator = PyTorch(entry_point=model,\n",
    "                    role=role,\n",
    "                    framework_version='1.4.0',\n",
    "                    py_version='py3',\n",
    "                    instance_count=1,\n",
    "                    instance_type='ml.g4dn.xlarge',\n",
    "                    base_job_name=model_save_name,\n",
    "                    output_path=model_save_path,\n",
    "#                     code_location=custom_code_uri,\n",
    "                    metric_definitions=[\n",
    "                        {'Name': 'train:loss', 'Regex': 'train Loss: (.*?);'},\n",
    "                        {'Name': 'train:acc', 'Regex': 'train Acc: (.*?);'},\n",
    "                        {'Name': 'validation:loss', 'Regex': 'validation Loss: (.*?);'},\n",
    "                        {'Name': 'validation:acc', 'Regex': 'validation Acc: (.*?);'},\n",
    "                        {'Name': 'learning_rate', 'Regex': 'lr: (.*?);'},\n",
    "                        {'Name': 'epoch', 'Regex': 'epoch: (.*?);'},\n",
    "                        {'Name': 'train_f1_score', 'Regex': 'train Avg. F1 Score: (.*?);'},\n",
    "                        {'Name': 'validation_f1_score', 'Regex': 'validation Avg. F1 Score: (.*?);'},\n",
    "                        {'Name': 'test_f1_score', 'Regex': 'test Avg. F1 Score: (.*?);'},\n",
    "                        {'Name': 'classification_report', 'Regex': 'classification_report: (.*?);'}\n",
    "                    ],\n",
    "                    hyperparameters={\n",
    "                        'image-height': height,\n",
    "                        'image-width': width,\n",
    "                        'batch-size': batch_size,\n",
    "                        'epochs': epochs,\n",
    "                        'lr': lr,\n",
    "                        'momentum': momentum,\n",
    "                        'T_0': T_0,\n",
    "                        'T_mult': T_mult,\n",
    "                        'num-layers': num_layers,\n",
    "                        'pretrained-weights': pretrained_weights,\n",
    "                        's3-bucket': bucket,\n",
    "                        'warm-restart': saved_model_path,\n",
    "                        'unfreeze-all-layers': unfreeze_all_layers,\n",
    "                        'train-augmentation': train_augmentation\n",
    "                    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-06-24 13:14:06 Starting - Starting the training job...\n",
      "2021-06-24 13:14:07 Starting - Launching requested ML instancesProfilerReport-1624540445: InProgress\n",
      "...\n",
      "2021-06-24 13:14:56 Starting - Preparing the instances for training.........\n",
      "2021-06-24 13:16:31 Downloading - Downloading input data...\n",
      "2021-06-24 13:17:01 Training - Downloading the training image......\n",
      "2021-06-24 13:18:07 Training - Training image download completed. Training in progress..\u001b[34mbash: cannot set terminal process group (-1): Inappropriate ioctl for device\u001b[0m\n",
      "\u001b[34mbash: no job control in this shell\u001b[0m\n",
      "\u001b[34m2021-06-24 13:18:08,970 sagemaker-containers INFO     Imported framework sagemaker_pytorch_container.training\u001b[0m\n",
      "\u001b[34m2021-06-24 13:18:08,991 sagemaker_pytorch_container.training INFO     Block until all host DNS lookups succeed.\u001b[0m\n",
      "\u001b[34m2021-06-24 13:18:08,994 sagemaker_pytorch_container.training INFO     Invoking user training script.\u001b[0m\n",
      "\u001b[34m2021-06-24 13:18:09,284 sagemaker-containers INFO     Module default_user_module_name does not provide a setup.py. \u001b[0m\n",
      "\u001b[34mGenerating setup.py\u001b[0m\n",
      "\u001b[34m2021-06-24 13:18:09,284 sagemaker-containers INFO     Generating setup.cfg\u001b[0m\n",
      "\u001b[34m2021-06-24 13:18:09,284 sagemaker-containers INFO     Generating MANIFEST.in\u001b[0m\n",
      "\u001b[34m2021-06-24 13:18:09,284 sagemaker-containers INFO     Installing module with the following command:\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/python3.6 -m pip install . \u001b[0m\n",
      "\u001b[34mProcessing /tmp/tmpzzanefie/module_dir\u001b[0m\n",
      "\u001b[34mBuilding wheels for collected packages: default-user-module-name\n",
      "  Building wheel for default-user-module-name (setup.py): started\u001b[0m\n",
      "\u001b[34m  Building wheel for default-user-module-name (setup.py): finished with status 'done'\n",
      "  Created wheel for default-user-module-name: filename=default_user_module_name-1.0.0-py2.py3-none-any.whl size=11134 sha256=0664c46ba33668a3829c088729fc47a93c460d4a7bf447e9230cda7ffb17a463\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-2hojcljv/wheels/80/da/97/f57604d95e79d123baa578e6cc1c9a2d834536fd641a4329e7\u001b[0m\n",
      "\u001b[34mSuccessfully built default-user-module-name\u001b[0m\n",
      "\u001b[34mInstalling collected packages: default-user-module-name\u001b[0m\n",
      "\u001b[34mSuccessfully installed default-user-module-name-1.0.0\u001b[0m\n",
      "\u001b[34m2021-06-24 13:18:11,663 sagemaker-containers INFO     Invoking user script\n",
      "\u001b[0m\n",
      "\u001b[34mTraining Env:\n",
      "\u001b[0m\n",
      "\u001b[34m{\n",
      "    \"additional_framework_parameters\": {},\n",
      "    \"channel_input_dirs\": {\n",
      "        \"test\": \"/opt/ml/input/data/test\",\n",
      "        \"validation\": \"/opt/ml/input/data/validation\",\n",
      "        \"train\": \"/opt/ml/input/data/train\"\n",
      "    },\n",
      "    \"current_host\": \"algo-1\",\n",
      "    \"framework_module\": \"sagemaker_pytorch_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"hyperparameters\": {\n",
      "        \"num-layers\": 18,\n",
      "        \"image-width\": 448,\n",
      "        \"T_0\": 225,\n",
      "        \"lr\": 0.01,\n",
      "        \"train-augmentation\": false,\n",
      "        \"unfreeze-all-layers\": false,\n",
      "        \"pretrained-weights\": true,\n",
      "        \"momentum\": 0.9,\n",
      "        \"batch-size\": 4,\n",
      "        \"warm-restart\": null,\n",
      "        \"image-height\": 448,\n",
      "        \"s3-bucket\": null,\n",
      "        \"epochs\": 15,\n",
      "        \"T_mult\": 1\n",
      "    },\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {\n",
      "        \"test\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        },\n",
      "        \"validation\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        },\n",
      "        \"train\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        }\n",
      "    },\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"is_master\": true,\n",
      "    \"job_name\": \"ResNet-cars-open-set-2021-06-24-13-14-05-399\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://custom-labels-console-eu-west-1-52749662e9/ResNet-cars-open-set-2021-06-24-13-14-05-399/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"resnet_sgd_cosineannealing_inference\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 4,\n",
      "    \"num_gpus\": 1,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-1\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\"\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"resnet_sgd_cosineannealing_inference.py\"\u001b[0m\n",
      "\u001b[34m}\n",
      "\u001b[0m\n",
      "\u001b[34mEnvironment variables:\n",
      "\u001b[0m\n",
      "\u001b[34mSM_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[34mSM_HPS={\"T_0\":225,\"T_mult\":1,\"batch-size\":4,\"epochs\":15,\"image-height\":448,\"image-width\":448,\"lr\":0.01,\"momentum\":0.9,\"num-layers\":18,\"pretrained-weights\":true,\"s3-bucket\":null,\"train-augmentation\":false,\"unfreeze-all-layers\":false,\"warm-restart\":null}\u001b[0m\n",
      "\u001b[34mSM_USER_ENTRY_POINT=resnet_sgd_cosineannealing_inference.py\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_PARAMS={}\u001b[0m\n",
      "\u001b[34mSM_RESOURCE_CONFIG={\"current_host\":\"algo-1\",\"hosts\":[\"algo-1\"],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[34mSM_INPUT_DATA_CONFIG={\"test\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"train\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"validation\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}}\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[34mSM_CHANNELS=[\"test\",\"train\",\"validation\"]\u001b[0m\n",
      "\u001b[34mSM_CURRENT_HOST=algo-1\u001b[0m\n",
      "\u001b[34mSM_MODULE_NAME=resnet_sgd_cosineannealing_inference\u001b[0m\n",
      "\u001b[34mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_MODULE=sagemaker_pytorch_container.training:main\u001b[0m\n",
      "\u001b[34mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[34mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[34mSM_NUM_CPUS=4\u001b[0m\n",
      "\u001b[34mSM_NUM_GPUS=1\u001b[0m\n",
      "\u001b[34mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[34mSM_MODULE_DIR=s3://custom-labels-console-eu-west-1-52749662e9/ResNet-cars-open-set-2021-06-24-13-14-05-399/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[34mSM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{\"test\":\"/opt/ml/input/data/test\",\"train\":\"/opt/ml/input/data/train\",\"validation\":\"/opt/ml/input/data/validation\"},\"current_host\":\"algo-1\",\"framework_module\":\"sagemaker_pytorch_container.training:main\",\"hosts\":[\"algo-1\"],\"hyperparameters\":{\"T_0\":225,\"T_mult\":1,\"batch-size\":4,\"epochs\":15,\"image-height\":448,\"image-width\":448,\"lr\":0.01,\"momentum\":0.9,\"num-layers\":18,\"pretrained-weights\":true,\"s3-bucket\":null,\"train-augmentation\":false,\"unfreeze-all-layers\":false,\"warm-restart\":null},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{\"test\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"train\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"validation\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}},\"input_dir\":\"/opt/ml/input\",\"is_master\":true,\"job_name\":\"ResNet-cars-open-set-2021-06-24-13-14-05-399\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://custom-labels-console-eu-west-1-52749662e9/ResNet-cars-open-set-2021-06-24-13-14-05-399/source/sourcedir.tar.gz\",\"module_name\":\"resnet_sgd_cosineannealing_inference\",\"network_interface_name\":\"eth0\",\"num_cpus\":4,\"num_gpus\":1,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_host\":\"algo-1\",\"hosts\":[\"algo-1\"],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"resnet_sgd_cosineannealing_inference.py\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ARGS=[\"--T_0\",\"225\",\"--T_mult\",\"1\",\"--batch-size\",\"4\",\"--epochs\",\"15\",\"--image-height\",\"448\",\"--image-width\",\"448\",\"--lr\",\"0.01\",\"--momentum\",\"0.9\",\"--num-layers\",\"18\",\"--pretrained-weights\",\"True\",\"--s3-bucket\",\"\",\"--train-augmentation\",\"False\",\"--unfreeze-all-layers\",\"False\",\"--warm-restart\",\"\"]\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_TEST=/opt/ml/input/data/test\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_VALIDATION=/opt/ml/input/data/validation\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_TRAIN=/opt/ml/input/data/train\u001b[0m\n",
      "\u001b[34mSM_HP_NUM-LAYERS=18\u001b[0m\n",
      "\u001b[34mSM_HP_IMAGE-WIDTH=448\u001b[0m\n",
      "\u001b[34mSM_HP_T_0=225\u001b[0m\n",
      "\u001b[34mSM_HP_LR=0.01\u001b[0m\n",
      "\u001b[34mSM_HP_TRAIN-AUGMENTATION=false\u001b[0m\n",
      "\u001b[34mSM_HP_UNFREEZE-ALL-LAYERS=false\u001b[0m\n",
      "\u001b[34mSM_HP_PRETRAINED-WEIGHTS=true\u001b[0m\n",
      "\u001b[34mSM_HP_MOMENTUM=0.9\u001b[0m\n",
      "\u001b[34mSM_HP_BATCH-SIZE=4\u001b[0m\n",
      "\u001b[34mSM_HP_WARM-RESTART=\u001b[0m\n",
      "\u001b[34mSM_HP_IMAGE-HEIGHT=448\u001b[0m\n",
      "\u001b[34mSM_HP_S3-BUCKET=\u001b[0m\n",
      "\u001b[34mSM_HP_EPOCHS=15\u001b[0m\n",
      "\u001b[34mSM_HP_T_MULT=1\u001b[0m\n",
      "\u001b[34mPYTHONPATH=/opt/ml/code:/opt/conda/bin:/opt/conda/lib/python36.zip:/opt/conda/lib/python3.6:/opt/conda/lib/python3.6/lib-dynload:/opt/conda/lib/python3.6/site-packages\n",
      "\u001b[0m\n",
      "\u001b[34mInvoking script with the following command:\n",
      "\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/python3.6 resnet_sgd_cosineannealing_inference.py --T_0 225 --T_mult 1 --batch-size 4 --epochs 15 --image-height 448 --image-width 448 --lr 0.01 --momentum 0.9 --num-layers 18 --pretrained-weights True --s3-bucket  --train-augmentation False --unfreeze-all-layers False --warm-restart \n",
      "\n",
      "\u001b[0m\n",
      "\u001b[34mGPU NAME: Tesla T4\u001b[0m\n",
      "\u001b[34mEpoch 1/15\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34m[2021-06-24 13:18:16.981 algo-1:43 INFO json_config.py:90] Creating hook from json_config at /opt/ml/input/config/debughookconfig.json.\u001b[0m\n",
      "\u001b[34m[2021-06-24 13:18:16.981 algo-1:43 INFO hook.py:192] tensorboard_dir has not been set for the hook. SMDebug will not be exporting tensorboard summaries.\u001b[0m\n",
      "\u001b[34m[2021-06-24 13:18:16.981 algo-1:43 INFO hook.py:237] Saving to /opt/ml/output/tensors\u001b[0m\n",
      "\u001b[34m[2021-06-24 13:18:16.981 algo-1:43 INFO state_store.py:67] The checkpoint config file /opt/ml/input/config/checkpointconfig.json does not exist.\u001b[0m\n",
      "\u001b[34m[2021-06-24 13:18:16.993 algo-1:43 INFO hook.py:382] Monitoring the collections: losses\u001b[0m\n",
      "\u001b[34m[2021-06-24 13:18:16.993 algo-1:43 INFO hook.py:443] Hook is writing from the hook with pid: 43\n",
      "\u001b[0m\n",
      "\u001b[34mtrain Loss: 1.9779; train Acc: 0.7282;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 0.4412; validation Acc: 0.8947;\u001b[0m\n",
      "\u001b[34mepoch: 1; lr: 0.01;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 2/15\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 1.1777; train Acc: 0.8021;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 0.2874; validation Acc: 0.9263;\u001b[0m\n",
      "\u001b[34mepoch: 2; lr: 0.01;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 3/15\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 1.3555; train Acc: 0.7966;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 0.2695; validation Acc: 0.9368;\u001b[0m\n",
      "\u001b[34mepoch: 3; lr: 0.01;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 4/15\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 1.8012; train Acc: 0.7827;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 0.2523; validation Acc: 0.9421;\u001b[0m\n",
      "\u001b[34mepoch: 4; lr: 0.01;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 5/15\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 1.3604; train Acc: 0.8299;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 0.2683; validation Acc: 0.9474;\u001b[0m\n",
      "\u001b[34mepoch: 5; lr: 0.01;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 6/15\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 1.7015; train Acc: 0.7949;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 0.2375; validation Acc: 0.9474;\u001b[0m\n",
      "\u001b[34mepoch: 6; lr: 0.01;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 7/15\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 1.5322; train Acc: 0.8266;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 0.3107; validation Acc: 0.9316;\u001b[0m\n",
      "\u001b[34mepoch: 7; lr: 0.01;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 8/15\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 1.5857; train Acc: 0.8121;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 0.1989; validation Acc: 0.9316;\u001b[0m\n",
      "\u001b[34mepoch: 8; lr: 0.01;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 9/15\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 1.3106; train Acc: 0.8349;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 0.2862; validation Acc: 0.9368;\u001b[0m\n",
      "\u001b[34mepoch: 9; lr: 0.01;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 10/15\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 1.7291; train Acc: 0.8093;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 0.2656; validation Acc: 0.9579;\u001b[0m\n",
      "\u001b[34mepoch: 10; lr: 0.01;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 11/15\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 1.4666; train Acc: 0.8188;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 0.3011; validation Acc: 0.9474;\u001b[0m\n",
      "\u001b[34mepoch: 11; lr: 0.01;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 12/15\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 1.4405; train Acc: 0.8421;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 0.4654; validation Acc: 0.9263;\u001b[0m\n",
      "\u001b[34mepoch: 12; lr: 0.01;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 13/15\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 1.6531; train Acc: 0.8116;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 0.2339; validation Acc: 0.9526;\u001b[0m\n",
      "\u001b[34mepoch: 13; lr: 0.01;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 14/15\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 1.9953; train Acc: 0.8088;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 0.2846; validation Acc: 0.9368;\u001b[0m\n",
      "\u001b[34mepoch: 14; lr: 0.01;\n",
      "\u001b[0m\n",
      "\u001b[34mEpoch 15/15\u001b[0m\n",
      "\u001b[34m----------\u001b[0m\n",
      "\u001b[34mtrain Loss: 1.4115; train Acc: 0.8460;\u001b[0m\n",
      "\u001b[34mvalidation Loss: 0.5724; validation Acc: 0.9421;\u001b[0m\n",
      "\u001b[34mepoch: 15; lr: 0.01;\n",
      "\u001b[0m\n",
      "\u001b[34mTraining complete in 16m 32s\u001b[0m\n",
      "\u001b[34mBest validation Acc: 0.957895\u001b[0m\n",
      "\u001b[34m/opt/ml/model/model.pth\n",
      "\u001b[0m\n",
      "\u001b[34mEvaluating best weights:\u001b[0m\n",
      "\u001b[34m--------------------\u001b[0m\n",
      "\u001b[34mtrain Loss: 0.2404 Acc: 0.9589\u001b[0m\n",
      "\u001b[34mtrain Avg. F1 Score: 0.959;\u001b[0m\n",
      "\u001b[34mclassification_report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    mercedes       0.97      0.95      0.96       899\u001b[0m\n",
      "\u001b[34mnon-mercedes       0.95      0.97      0.96       900\n",
      "\n",
      "    accuracy                           0.96      1799\n",
      "   macro avg       0.96      0.96      0.96      1799\u001b[0m\n",
      "\u001b[34mweighted avg       0.96      0.96      0.96      1799\u001b[0m\n",
      "\u001b[34m;\n",
      "\u001b[0m\n",
      "\u001b[34mvalidation Loss: 0.2656 Acc: 0.9579\u001b[0m\n",
      "\u001b[34mvalidation Avg. F1 Score: 0.958;\u001b[0m\n",
      "\u001b[34mclassification_report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    mercedes       1.00      0.92      0.96        95\u001b[0m\n",
      "\u001b[34mnon-mercedes       0.92      1.00      0.96        95\n",
      "\n",
      "    accuracy                           0.96       190\n",
      "   macro avg       0.96      0.96      0.96       190\u001b[0m\n",
      "\u001b[34mweighted avg       0.96      0.96      0.96       190\u001b[0m\n",
      "\u001b[34m;\n",
      "\u001b[0m\n",
      "\n",
      "2021-06-24 13:36:16 Uploading - Uploading generated training model\u001b[34mtest Loss: 0.4124 Acc: 0.9227\u001b[0m\n",
      "\u001b[34mtest Avg. F1 Score: 0.923;\u001b[0m\n",
      "\u001b[34mclassification_report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    mercedes       0.94      0.90      0.92       220\u001b[0m\n",
      "\u001b[34mnon-mercedes       0.90      0.95      0.92       220\n",
      "\n",
      "    accuracy                           0.92       440\n",
      "   macro avg       0.92      0.92      0.92       440\u001b[0m\n",
      "\u001b[34mweighted avg       0.92      0.92      0.92       440\u001b[0m\n",
      "\u001b[34m;\n",
      "\u001b[0m\n",
      "\u001b[34m2021-06-24 13:36:09,002 sagemaker-containers INFO     Reporting training SUCCESS\u001b[0m\n",
      "\n",
      "2021-06-24 13:36:36 Completed - Training job completed\n",
      "ProfilerReport-1624540445: IssuesFound\n",
      "Training seconds: 1204\n",
      "Billable seconds: 1204\n"
     ]
    }
   ],
   "source": [
    "estimator.fit({'train': train,\n",
    "               'validation': validation,\n",
    "               'test': test})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}