custom_models_and_training_with_tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Custom Models and Training with TensorFlow**\n",
    "\n",
    "\n",
    "This notebook is inspired from the handson-ml2 GitHub repository by Aurélien Geron\n",
    "\n",
    "https://github.com/ageron/handson-ml2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures. We also check that Python 3.5 or later is installed (although Python 2.x may work, it is deprecated so we strongly recommend you use Python 3 instead), as well as Scikit-Learn ≥0.20 and TensorFlow ≥2.0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Python ≥3.5 is required\n",
    "import sys\n",
    "assert sys.version_info >= (3, 5)\n",
    "\n",
    "# Scikit-Learn ≥0.20 is required\n",
    "import sklearn\n",
    "assert sklearn.__version__ >= \"0.20\"\n",
    "\n",
    "try:\n",
    "    # %tensorflow_version only exists in Colab.\n",
    "    %tensorflow_version 2.x\n",
    "    !pip install -U tqdm\n",
    "except Exception:\n",
    "    pass\n",
    "\n",
    "# TensorFlow ≥2.0 is required\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "assert tf.__version__ >= \"2.0\"\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "mpl.rc('axes', labelsize=14)\n",
    "mpl.rc('xtick', labelsize=12)\n",
    "mpl.rc('ytick', labelsize=12)\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"deep\"\n",
    "IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID)\n",
    "os.makedirs(IMAGES_PATH, exist_ok=True)\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n",
    "    path = os.path.join(IMAGES_PATH, fig_id + \".\" + fig_extension)\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format=fig_extension, dpi=resolution)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensors and operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=42>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Scalar Tensor of integer\n",
    "\n",
    "tf.constant(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=42.1>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Scalar Tensor of Float\n",
    "\n",
    "\n",
    "tf.constant(42.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tensor representing matrix with two rows and three columns of floats\n",
    "\n",
    "tf.constant([[1., 2., 3.], [4., 5., 6.]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([2, 3])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check shape of Tensor\n",
    "\n",
    "t = tf.constant([[1., 2., 3.], [4., 5., 6.]])\n",
    "t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tf.float32"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check data type of Tensor\n",
    "\n",
    "t.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[ 6.,  7.,  8.],\n",
       "       [ 9., 10., 11.]], dtype=float32)>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tensor Operation - Addition\n",
    "\n",
    "t + 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[ 6.,  7.,  8.],\n",
       "       [ 9., 10., 11.]], dtype=float32)>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tensor Operation - Addition - Another way\n",
    "\n",
    "\n",
    "tf.add(t,5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[ 1.,  4.,  9.],\n",
       "       [16., 25., 36.]], dtype=float32)>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tensor Operation - Square\n",
    "\n",
    "tf.square(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[ 5., 10., 15.],\n",
       "       [20., 25., 30.]], dtype=float32)>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tensor Operation - Multiply\n",
    "\n",
    "tf.multiply(t, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[1.       , 1.4142135, 1.7320508],\n",
       "       [2.       , 2.236068 , 2.4494898]], dtype=float32)>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tensor Operation - Square root\n",
    "\n",
    "tf.sqrt(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 2), dtype=float32, numpy=\n",
       "array([[1., 4.],\n",
       "       [2., 5.],\n",
       "       [3., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tensor Operation - Transpose\n",
    "\n",
    "tf.transpose(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 2), dtype=float32, numpy=\n",
       "array([[14., 32.],\n",
       "       [32., 77.]], dtype=float32)>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tensor Operation - Matrix Multiplication\n",
    "\n",
    "tf.matmul(t, tf.transpose(t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 2), dtype=float32, numpy=\n",
       "array([[14., 32.],\n",
       "       [32., 77.]], dtype=float32)>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# # Tensor Operation - Matrix Multiplication - Another way\n",
    "\n",
    "t @ tf.transpose(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## From/To NumPy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=float64, numpy=array([2., 4., 5.])>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Creating tensor from NumPy array\n",
    "\n",
    "a = np.array([2., 4., 5.])\n",
    "tf.constant(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert tensor to Numpy array\n",
    "\n",
    "t.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert tensor to Numpy array - Another way\n",
    "\n",
    "np.array(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=float64, numpy=array([ 4., 16., 25.])>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Creating tensor from NumPy array\n",
    "\n",
    "tf.square(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  4.,  9.],\n",
       "       [16., 25., 36.]], dtype=float32)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert tensor to Numpy array\n",
    "\n",
    "np.square(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conflicting Types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adding tensors of float and integer\n",
    "\n",
    "# tf.constant(2.0) + tf.constant(40)\n",
    "\n",
    "# The above code will throw exception"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cannot compute AddV2 as input #1(zero-based) was expected to be a float tensor but is a int32 tensor [Op:AddV2] name: add/\n"
     ]
    }
   ],
   "source": [
    "# Adding tensors of float and integer\n",
    "# Handling Exception\n",
    "\n",
    "try:\n",
    "    tf.constant(2.0) + tf.constant(40)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cannot compute AddV2 as input #1(zero-based) was expected to be a float tensor but is a double tensor [Op:AddV2] name: add/\n"
     ]
    }
   ],
   "source": [
    "# Adding tensors of 32-bit and 64-bit\n",
    "# Handling Exception\n",
    "\n",
    "try:\n",
    "    tf.constant(2.0) + tf.constant(40., dtype=tf.float64)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=42.0>"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Type Casting\n",
    "# Converting 64-bit tensor to 32-bit and then adding tensors\n",
    "\n",
    "t2 = tf.constant(40., dtype=tf.float64)\n",
    "tf.constant(2.0) + tf.cast(t2, tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'Variable:0' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tf variable with two rows and three columns\n",
    "\n",
    "v = tf.Variable([[1., 2., 3.], [4., 5., 6.]])\n",
    "v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2.,  4.,  6.],\n",
       "       [ 8., 10., 12.]], dtype=float32)>"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Modify in place with assign()\n",
    "\n",
    "v.assign(2 * v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2., 42.,  6.],\n",
       "       [ 8., 10., 12.]], dtype=float32)>"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Update cell with assign()\n",
    "# Update cells with index (0,1) to 42.0\n",
    "\n",
    "v[0, 1].assign(42.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[100.,  42.,   6.],\n",
       "       [  8.,  10., 200.]], dtype=float32)>"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Update cell with scatter_nd_update()\n",
    "# Update cells with index (0,0) and (1,2) to 100.0 and 200.0 respectively\n",
    "\n",
    "v.scatter_nd_update(indices=[[0, 0], [1, 2]], updates=[100., 200.])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Other Data Structures - Strings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=string, numpy=b'hello world'>"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# String Tensor - Byte String\n",
    "\n",
    "tf.constant(b\"hello world\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=string, numpy=b'caf\\xc3\\xa9'>"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# String Tensor - Unicode strings get encoded to utf-8 automatically\n",
    "\n",
    "tf.constant(\"café\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=int32, numpy=array([ 99,  97, 102, 233], dtype=int32)>"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Represent Unicode strings using tensor of type tf.int32\n",
    "\n",
    "u = tf.constant([ord(c) for c in \"café\"])\n",
    "u"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## String arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=string, numpy=\n",
       "array([b'Caf\\xc3\\xa9', b'Coffee', b'caff\\xc3\\xa8',\n",
       "       b'\\xe5\\x92\\x96\\xe5\\x95\\xa1'], dtype=object)>"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tensor of string arrays\n",
    "\n",
    "p = tf.constant([\"Café\", \"Coffee\", \"caffè\", \"咖啡\"])\n",
    "p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ragged tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.RaggedTensor [[b'All', b'the', b'world', b'is', b'a', b'stage'], [b'And', b'all', b'the', b'men', b'and', b'women', b'merely', b'players'], [b'They', b'have', b'their', b'exits', b'and', b'their', b'entrances']]>"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# List of lists, with each being of variable length\n",
    "# Consider a speech like this, where the length may change wildly on each line\n",
    "\n",
    "speech = tf.ragged.constant(\n",
    "  [['All', 'the', 'world', 'is', 'a', 'stage'],\n",
    "  ['And', 'all', 'the', 'men', 'and', 'women', 'merely', 'players'],\n",
    "  ['They', 'have', 'their', 'exits', 'and', 'their', 'entrances']])\n",
    "\n",
    "speech"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([b'All' b'the' b'world' b'is' b'a' b'stage'], shape=(6,), dtype=string)\n"
     ]
    }
   ],
   "source": [
    "print(speech[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([b'And' b'all' b'the' b'men' b'and' b'women' b'merely' b'players'], shape=(8,), dtype=string)\n"
     ]
    }
   ],
   "source": [
    "print(speech[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.RaggedTensor [[b'Matt Damon', b'Robin Williams'], [b'John Travolta', b'Samuel L. Jackson', b'Uma Thurman', b'Harvey Keitel', b'Bruce Willis'], [b'Tim Robbins', b'Morgan Freeman', b'Bob Gunton', b'William Sadler']]>"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# One more example of Ragged Tensors\n",
    "# Variable-length features, such as the set of actors in a movie.\n",
    "\n",
    "actors = tf.ragged.constant(\n",
    "  [['Matt Damon', 'Robin Williams'],\n",
    "  ['John Travolta', 'Samuel L. Jackson', 'Uma Thurman', 'Harvey Keitel', 'Bruce Willis'],\n",
    "  ['Tim Robbins', 'Morgan Freeman', 'Bob Gunton', 'William Sadler']])\n",
    "\n",
    "actors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([b'Matt Damon' b'Robin Williams'], shape=(2,), dtype=string)\n"
     ]
    }
   ],
   "source": [
    "print(actors[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[b'John Travolta' b'Samuel L. Jackson' b'Uma Thurman' b'Harvey Keitel'\n",
      " b'Bruce Willis'], shape=(5,), dtype=string)\n"
     ]
    }
   ],
   "source": [
    "print(actors[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Custom loss function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's start by loading and preparing the California housing dataset. We first load it, then split it into a training set, a validation set and a test set, and finally we scale it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use fetch_california_housing() function to load the data. \n",
    "# This dataset contains only numerical features (there is no ocean_proximity feature)\n",
    "# And there is no missing value. \n",
    "# After loading the data, we split it into a training set, a validation set, and a test set\n",
    "# And we scale all the features\n",
    "\n",
    "\n",
    "from sklearn.datasets import fetch_california_housing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# fetch the data\n",
    "housing = fetch_california_housing()\n",
    "\n",
    "# split it into a training set, a validation set, and a test set\n",
    "X_train_full, X_test, y_train_full, y_test = train_test_split(\n",
    "    housing.data, housing.target.reshape(-1, 1), random_state=42)\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(\n",
    "    X_train_full, y_train_full, random_state=42)\n",
    "\n",
    "# Scale all the features\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_valid_scaled = scaler.transform(X_valid)\n",
    "X_test_scaled = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Loss Function - Huber Loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's implement huber loss. Huber loss is less sensitive to outliers in data than mean squared error.\n",
    "Below is the formula of huber loss."
   ]
  },
  {
   "attachments": {
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAR8AAAA8CAYAAABfGL5hAAAQkElEQVR4Ae3d6c0tOREG4IYIIAO2AIAEWP4ghJCA/4glAWASYEkAiIAlASACIAIgAiACyAD0jM6rqfG4Ty/u891zv+uSenza7SqXq8qvy+7+7izLpGmBaYFpgWmBaYFpgWmBaYFpgb0W+PmyLL/a23i2mxaYFpgWGLXA55dl+duyLH9fluVTB4V9YlmWbx3kmc2nBaYFpgUWwPOfG/AAkr2ET5b0r2VZ/rKXababFpgWmBaIBYDHf5dl+XQqDpa2aleAj4wLmP32BmhfPqjHbD4tMC3wFlngB8uy/G9Zlt8N6HwV+PxxWd7Pwqjy/Vs2diQTGxjCZH2ABWTGR7fwUWOENzJm+eQW+OsNfL49oOdV4CMDk/UgoAMUZ/ZzM8jFhUXnl8uy/OhiuVWcuPhKrTjwe4T3QDerTc2HP98udgKGky62gAnu+uSA3KvAp6rA2fQ6u3JWWfP3BxYA6l4sAPkfPxjgRwBkhPeD0X7wSxztjaWPlWOEn9yOJNhq0oUWYOSAj99n6RHg4wxpOvysR9b5TCY+RwDeVnfvpLyx7S5GAGSEtyroHPM3NzDZm70AaFn4yJyoOszfKxZ4RvD52e3MZ0XlWT1ggT8ty2Kr/RI0AiAjvMZmuye7c33hxGCBz3sn+CbLTguMZj5WzJ/e9sX/vP0eXUUdNLuQlWrvanVjmcWGBWSUV7yZ3Ojm/ccjAHKWF+g4q/EC5ezbW8pbAC3MZ4Brj23e+Taj4CM9dSDM4UqXunvkoPMPN+cCqnqgDHScRwA0zhdEE3zuWfPYMxPaIuFiX74IZXviUwdbsSwAnvOTQ1f+8Pw7t99k3KM1AHG+aEtd46D18xrvWn/GQu9R0CGffgEf8TjpARYYBZ8jKtWDTny+iLaySG9Dgicrc8o8m+W4Bb56+5DUV+x+f/Em0urOD1nl+cp2xVkJcu/tj2/BfIwKGPiH/+5luj0AIUv/zp4QkCO33Qr2eG8sHyrEEXnaj7w0iVAgCJzpR6YxVpBOu1kOWuAlwcdZQwUaqgu6ke+LBof/TrIH1Pk+ZPICm0pAxcSrn2AEcACISbr1QqAHIPoip5J+fl0rbhN/z2v6gI9MZRR88FdgjA1a3RpV5+0ZC5wFn693MpQEdcpvFIUEsADLaueRNL+tKyzz54MsEP8EfGQ7/AAoWlJv0QjhNTnDm/q1sgUfGQSZFdAATFtHXsu71kfqyabbCAgBmXaBpNtcIGPlC8uz4PPZZVm+u3F9rujJqZxYU/QEYlL90nz+fKAFAj7pIpN/DXy0D7W8qV8rWwABZOJA5hTSRl2btbS8ab9VGo/tu0zu6IEz8HKmVYluPdvUNvP3CQucBZ+jXQlaW6y6YgqO1NVD56OyZ/tjFgiAxBdrmU9io818Khht9dwCCF4TvJKzntTVfxmh5a08e34DIf0dASFA0x60qyNr0sUWSIAxcILx4i7eFydtrUFr5XOoJ7D1m+B7RN9T5octwA/tG5xenQPpduJpV/34YckfvWsBpOUXB/qQGct8ZCyhljf1R0vgKv5cNfPuybHlqkcDsvN2vLZ13tLV7K0na9ZtWOAK8HHwyCGuNYcktddWMHAoR0txOZuTJz3eAnwE9E14nzPE7iblP27+4CtnMgAqE9Hz8OLHuycbaAGEXNkuebJdgOBeHGhbz4Ja3lHr2IKJv3vEHhZCMUo/Mep3JfqyX2xTn73Y762BtIow+NrkbNu+1P0o+AR4jMubD45ZG6NgFVAuqxx7+L31xuSlbPEu9BP7pwz4GDufuM+zOukCPnmm3LNV1q4FqcSByStWxJDMp2536NPjfQkfVf2MuyX62srRb5NMMBdKebs9Xej8KPhwrhVf+SxUweeMTlkNw2ul2OWUMMzyVVtgBEBGeB9tVCBdgbvbH3SWRkmTcgGA9kS7y7xSma3CyuO71fnI626jF3xYwecMMAPgumKxNUCaNC3AAiMAspfXfDSn91w1Vkc85GxqLcP/iFyKAZ9eGvWRxncqpGRW97Nkgjtk/eFZAQ/gCyiPigZE2c+Pypr8r8MCewGkN9ojvObVnqvXz9E6xwSHQMwEAxpnVveqHBAbPWgKgI3qUvU6+5subFNfp56RZRVgmz3nAGfkT5630wIA5GxMjPA+jbUcnJlgTtZHSNZEztEPl9o+gY4MoZ7st21e6j4f/23uXzcUkvrmDGw0u9zoaj6eFnh7LPDeDTRGJ5hUC2jcI0BX07LeRAQ+Pqwy8d8kBQTvvaHao1+AhzwZkBVr0rTAtMBtS3FFxmJS2VqsEZBz4GorA4Bs8/RrUra0JSvtgZm0de8Vvj1lsp76SnUPX23jcNkY6/UMGV3Vcf6eFnhjFrCy7z0ktnXw9WLvH4yWrayBj3OgNoPQFhj1CPjsOYPS7hcHrl6m1etfFggwRrNB9gK29dr9FqCn2KybFngtFtg676l/S2LMthDIFqt9BkyAQUvOgPwbJ779Ccl2TO61rRWw8ryXFUXGo8oAH8CYNC0wLfAgC2yd97SZCQBZO51fA58ASd1umNjApdbVIYbnTYLP2jirnvP3tMC0wEkL+BgICPTeUNlytJmMCdlun9K119G9bZe3aPqoXy4H9NT1tkL6xbNF2jlX2Xv1+ur1cdW2qyd71k0LvPMWyNand97jrMJWqYKSOtsu2VDdQsWQAKAHPj0g0S5ZVcrIUeLpyapt/LZt9FX03uvIeYstIRuMHDi3+r7me/Hhf7DnDytn1viaPT0wNhPwS7dAkV2Y5ILFpd5f5Jp0dfLjARJKZz0CrCVbqF59tljJOkxq2RP5wK0FMqDo8Lqtb/t79D096MkWR0Dr0Xo9q3zg4xJTE3ye1UtvWC8gYCtk8pvkeUuVe6WrHigDjPx1tcDqZUu2UAKvZksZKmCyxdMvMBKk+gAw7cTOpB9905S+R0rjNqZn0GVkHC/Fm2x6gs9LWfwd6AdQJKCU9R81qsN37uOweISAk4yjnhGNyBvhTdY28ucVgBeQJ/Nr9QHEeYuYZyP9rfkmsh9ZHgUfC483pzLuLEJs/qaz3kfaaMo+aAHBkEzI7wBRK0bg9LKitt3aveA18Zz5PAvJfFxnSOaUszAyWmAAPNnOVvnAni3O0Ajvmf4qT8BHHGwRsGEP9rHYsE+ya/5/phjYGkueA1L/yJix7LFB+F6qZHPHKuLurSFKW50FSkBoTXlbq7PZz2v7JzUqEAvM1umApxekIwAywrvm0731R8DHuIEzEl9iS4nIYbuebW5N3nixthXn52cFHzbuLYJv3JhXKgCA2om2Jd82q5cFbPE98nkmE4edoXsTSPD6pzl7NAIgI7w9XY7UxV57QMM2VMYTwGn78TmG/wPos9Kane0KnhV8zEnZ5trO5VltfVivo+AjGNcC8XDnFzFkMgkmv4+SbNHbst75FeBZyxDXAntP/yO8e+RvtWGrvcENnNutaOTnBcYzfuoQ3Xox8czgE9vO8i2wwCj4ePNnMrYreIJ3LUO4ByB4703ue7yPNLmU3jlNLbf6yxalB8JsD5zWtjdbskeeWwQdL7jat7fs79yTXz9+W5QqCLXgAzzvAahnvo9qfereQXwWcbHSvrTQb/rO79wbf+pii1Zm6sn93m28fqfPPGcPvnK19kibWV5sAc4TZK7q1D3dmDS2kcAAfwUav+/J7AGIAPBHvTmk1Qao5ZA2OvV48+xZSmPxx8k+8wAwa9mhSW4L/5LEb2xoogEfPgSooejMf/lUpf7LmwEfcugPkPnMN3BtDKl36Uef2rANcq8P/XsJk3vgIKPOIb3niTXt2RN4qHPvooc6bd2TFRI/dBCT3sxqUw/72YFMpXr9kj3pwRY4Cz4JBoEkWDi8TiIrvbo1EhxtoAoKQRISDGRUUPOsxxueZynZwoRAgt84BH5Lgr1OlPb51fdZFPgsxIfthKMXnVsf4Qn4mLAhMrSvkzYAErAhSz+xC15jj20AFJnRLeCSrCp91CySnvWeTDKqTfVJ5xA/4EPJ3MVayLP54W2s8cDyDPgIBgFTQaF1OAdyei94DacFEO3JTOBpkzrbgEotb32W3/rde4VnrfzasizfvHNVO5BB7zoxrcjGpr4ldUB3zU5t+9F7ttNfS+pNuFBs39Mr4FPHo50xJoPKfXzl3gV46l8JeI7Ps5aydc8bwwBF/R8eOlfsxQe5IXGIx7Yr261srfKR7WdKvHzxplPr18ib5UUWSJCsBUCvGw5rA5izq8MFZg2SVo62CTilAGllSvnbOnIqbyvXPQDLdmGr3POxY/SgS++q2ZqxAJ5MmOjHvu0K7Vkmedo9soyvq5/Snzo6ZiJHr/go7ZT3wCeAlAyLXHXtFXme31uk+CdgxX6/v+nJxwCkZttk0pdMV0hWow/jc5EXEOJbda1+7mu2FFmzvNACCUgO6AVarysTUEpdyYTjsJBAuSdTcKS/rGh1wkavWhfZlTd1z1JmLHXVTOaT7UPVlc16AFvbXPWbTU3COjEjWx1/rYFP3U7dA59kPslaen2lT6Xn99rYItErZzUOwI1BfPlUobeV7cm0ZdNWPOHPwgjcyO/5puo5fz/AApnk94Cidqt9CzSZcMpQVj7teyRA8izbuLrvToALGLIS1GRV3p7sN1mXsdRV08St27CqH/BpJ59V3VugR5AszfYqtteH3wCw6kivGhP1nCa+qYsNGdrXOvLavvRX/dwDijpucgMWyXLoAjxau2Usrcw6Lm2cLdEV5Vypgqt6C0ayI/fGnLOr9xnnf8YtkKCpgbYlVYDVV+uAoc1QeoBU5QoQfYeymrnnZMFFJ3IEXQW2ljcynqU0kWswC/7eCm380v42izQ+Y68yrhobO7J1ZNMhC0XN1gAEHUw6WUzdWgZ86oKQTKeOJXJrHXCusWKsyULWxsj/dEl2wpbuKyBWXvYnN9Ta0thrthkdIl/8GW+ywLwwqOON7FkOWOAM+HCOAOJAAVCDKaqQ+++VSacNh2sTEtAm6W9ujrfqkA/kavBq3/JGxqNLE5Iuxky3ujLWvtXTUQADl0z02sbv2L59LsjJr1lEyztyD4Dopg96KjPxqlzPnI/UsdruqM/FFvTMvZIf41ty8QMYvkyseJ7+2QhfBbiqB7CpMj1zL4OpBOwis+qhLT3V+ZSjjge/eOZX7eioXfVtPhNo+6t9z98nLWBlcCVg9orR/h6PoOT0HnFwj7dX1/Kv8bbtrr4XtHkbZ1xbK/aWfUwWWYjgb8mkfRT4pK8t/dLuinKtrz3+PtN/T+6aDpG/9TztZnmRBRj8LPhsqSBtXZtcIwAywrul89pzoGMsMjRUzw1uVYeLZA09RsBTt0G9NrNuWuCtt0DAJ3vcKwdkEvVW8BEAGeG9amxJ08/K8y2JNL+X9Uj517YgZ/ubfNMCT2kBkxkAPWqldWCYjCEGGAGQEd70P1LKggBHPRc4Ig/IG8Mav/oeKB3pY7adFngrLJDT/LzKfITS9fUq+SOHdyO8o2MDCveAY4984LIGPHv4Z5tpgVdjAec+PlOX/eRA9dUM7uKB2G7FRu1bqou7muKmBd4NC1iJHah6gzNT/r7PHRADH6/C/cW6t1+TpgWmBS6wgNe7zjIAUFb3C8S+ChHskcPzlDPzeRWunYN4Fgs4DDW5rPCTpgWmBaYFpgWmBaYFpgWmBaYFpgWmBToW+D8Gz/ld5kHQDwAAAABJRU5ErkJggg=="
    }
   },
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](attachment:image.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Huber loss (green) vs squared error loss (blue)"
   ]
  },
  {
   "attachments": {
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAASwAAADhCAYAAAByfIirAAAgAElEQVR4Ae2d+VcWV5rH81/MmT792/w2032mk850pzvdpzOnM5lMcux00jPJdDIajSHingC+uC8YN4xLjJq4azBGowZ3URQj7qi4i2ggKKIIiAKCLK985zyFpS+XF96t6lYV9b3n6PveWu7z3M+t90vVrXvv8wyYSIAESMAjBJ7xiJ90kwRIgARAweJFQAIk4BkCFCzPNBUdJQESoGDxGiABEvAMAQqWZ5qKjpIACVCweA2QAAl4hgAFyzNNRUdJgAQoWLwGSIAEPEOAguWZpqKjJEACFCxeAyRAAp4hQMHyTFPRURIgAQoWrwESIAHPEKBgeaap6CgJkAAFi9cACZCAZwhQsDzTVHSUBEiAgsVrgARIwDMEKFieaSo6SgIkQMHiNUACJOAZAq4QrIMHD2LZsmW4e/euZ8DRURIgAf0EHBesW7duoW/fvigoKEBSUpJ+ArRIAiTgGQKOC1ZbWxuSk5MRCASwYcMGz4CjoyRAAvoJOC5YpaWlGDt2LIqLi5GammoQ+OMfk/DOO+/gww8/RHp6upZ/gwYN0mIntD60aV/b+oXt0KFDjd9N6HVl1/e0tDTjd/n8869g8+ZT+tUKcD4IxdWrV9G/f3/s2rXLECih8Pzz7+L6db089u/fr9cgANq0D7kTbPPy8tDe3m5fpcKUXFhYiNra2jB77Nv0l7+sxZo1a+0z0EPJjt9hiW9XrlzBvn370NjYaLjar99gLFjQg9c27Prxxx9tKLXnImmzZz6J7HWKrW7BqqioQFNTUyKoYjq3rQ3429+ykZ2dHdN5Vh3sCsFSKyOPiOnp6lbmSYAEnCawdy8wc+Y+4wbDCV9cK1jz5gEVFU4goU0SIIHuCIwZA+TkULA68ZE7rBs3gMWLO21mhgRIwEEC8jg4ahSMuyvpwnEiufYOS2DwsdCJS4I2SSA8gbw8YMcOClYXOnKHJWnOHKCysstubiABEnCAgDwONjdTsLqgNwWrtBT46qsuu7mBBEhAM4FgEHg8TJKPhCp7U7BkeyCg7mWeBEhAN4HDh4FNmzqsSv8V+7BCWiBUsDIzgaqqkJ38SgIkoJ2A9NI8fNhhloKl4A8VrKIiYOVK5QBmSYAEtBF49AhISXlqjoL1lIXxLVSwZAMfCxVAzJKARgIHDgBbtjw1SMF6ysL4pgrW7NnAnTvKQcySAAloIWC+HTSNUbBMEo8/VcGSt4VLligHMUsCJGA7ARksGvo4KAYpWAp2VbBkNx8LFUjMkoAGAjKgfdu2zoYoWJ15GOtjKZuMQaScW6hSYZ4E7CUwejTQ0tLZBgWrM4+wgiVzCxctUg5klgRIwDYC8jiYlta1eAqWwiTcI6EcwsdCBRSzJGAjAVlKZufOrgYoWAqT7gTr88+BmzeVg5klARKwhYAsPtDa2rVo3wuWuQZ1nz59DDrdCZYsm7xwYVeA3EICJGAtge4eB8WK7wVLIMja7pMnTzaodydYspOPhQYi/kcCthKQR8E9e8KboGABGDduHG5Izzpk3EcKSkpKUF1d3YWYrPVeXt5lMzeQAAlYSEA62+UuKzQFg0Hjd5mVleXvyc+tra3o16/fEzYSo1AigZhBKZ7sQMeI97lzQ7fwOwmQgJUEJBaMjG5XkwTYkN+lBKCQuywnkitWHJWoOXtC7j97eiQUSHwsdOJSoU2/EJB4xkeOdF9bPhIqbCIJ1qpVwMWLyknMkgAJWELgk0+AnsIrUrAUzJEEq64OmDZNOYlZEiCBhAncuwdMmtRzMRQshU8kwZLD5bGwp78CSpHMkgAJREFgxQrg3LmeD6RgKXyiEazNm4Fjx5QTmSUBEkiIwMcfRz6dgqUwikawJHrHhAnKicySAAnETeD2bYnqHPl0CpbCKBrBklNk6oA6VkQpilkSIIEoCcgYx5KSyAdTsBRG0QpWTo5ME1BOZpYESCAuAiNHRncaBUvhFK1gyeL4j2OuKiUwSwIkEAuBK1ein6dLwVLIRitYcpr0Y5nhh5RimCUBEoiSwPTp0UdZp2ApUGMRrFOngPXrlQKYJQESiJqADA+K5u2gWSAFyyTx+DMWwZJTpPOdiQRIID4CMg1HpuNEmyhYCqlYBUuiQ1dWKoUwSwIkEBUBmegcS7cKBUvBGqtgycJ+X3yhFMIsCZBARAIynjHcuu09nUjBUujEKlhy+qhRSiHMkgAJRCQgEZ3z8iIe1ukAClYnHDJUYayyJXJW5kBxBYfInHgECYQSkJUZZHhQLImCpdCKR7BkBYdPP1UKYpYESKBbAtGszBDuZAqWQiUewZIiuIKDApJZEuiBgDyVnD/fwwHd7PK9YFVVVWH16tW4cOGCgShewdq6FTh0qBvK3EwCJNCJgDwOxpN8LVgPHz7EgAEDcPnyZZSVlRn84hUsiaE2blw8TcBzSMBfBEpL43+z7mvBOnToEN5++21MmjQJOTKbGfF1upuXm6yW2NBg5vhJAiQQjoD099bWhtsTeZuvBSs/Px+jR482KIlwSRoyZAhyc3NRVFQUmZ5yhLwpXLNG2cgsCZDAEwLyVjDWsVdycktLi/G7zMzM9G/UnKamJrz//vsQ4UpOTjagxvtIaLZIPI1hnstPEujtBA4cAHbsiL+Wvr7DEmyVlZWGYjfLsNsEHwnlfAlnH81CZIYx/kcCPiMgU3ESWfjS94KlXi+J3mHdvQvMmKGWyjwJkID0706enBgHCpbCL1HBkuJSU2Mfwau4wSwJ9DoCX38NXLqUWLUoWAo/KwQrOxvIz1cKZpYEfE7AiqjpFCzlIrJCsOQZnetkKWCZ9TWBH38EvvwycQQULIWhFYIlRY4fzzFZClpmfUxA+nWlfzfRRMFSCFolWLKw/pIlSuHMkoAPCQSD1j1xULCUC8gqwZJiY1mrWnGDWRLoNQR27gT277emOhQshaOVgrVyJXDmjGKAWRLwGQErVzKhYCkXj5WC9eABJ0QreJn1GYE7d6ILQR8tFgqWQspKwZKi5a9LY6NihFkS8AmBefOA8nLrKkvBUlhaLVgFBcDatYoRZknABwQk5mC0IeijxUHBUkhZLVhSPDvfFcjM+oLAsWPAt99aW1UKlsLTDsGSMGBXryqGmCWBXk5ABk/LwpZWJgqWQtMOwaqpATIyFEPMkkAvJhBvkIlISChYCiE7BEtMpKQAj1ewUSwySwK9j8DnnwOyFLLViYKlELVLsI4cATZsUIwxSwK9kIB0to8YYU/FKFgKV7sES8zIsjNMJNDbCRw8CGzebE8tfS9Y7733HtLT07Ft2zaDsJ2CtWgRUFxsT0OyVBJwC4FRoxJbVbSnevhesPr37w+JntP2eN1WOwXr/n1g4sSemoP7SMDbBKqq7H3B5HvBkvBemzdvxkcffWRcKSkpKSgpKUF1dbUtV4789Xn40JaiWSgJOE5ARrbfuGG9G8Fg0PhdZmVl+TdqTijWvn37GtlAIIDa2lo02jSf5uhRYP36UMv8TgK9g4AdI9tNMu3t7cbvMjs727+CJZGf09LSkJqaiu3btxts7HwkNOFbPV3BLJefJOAkgX37gK1b7fXA94+EKl4dgvXVV0AccVpVV5knAVcR+OQTQBbrszNRsBS6OgSrqYnLzijYmfU4gbIyYPZs+ytBwVIY6xAsMTlhAiBvDZlIoDcQmDoVsOk9VSc8FKxOOABdgiVRRBYsUIwzSwIeJCBTzmTqmY5EwVIo6xIsMTt0KAOuKviZ9SCBdesAefutI1GwFMo6BUsCrsqbFSYS8DIBnW+9KVjKlaJTsOSNil2TRJVqMUsCthC4eBFYutSWosMWSsFSsOgULDEtb1ZKShQnmCUBjxAYM0bvskkULOXC0C1YElWE8wuVRmDWEwQkkvOUKXpdpWApvHULlpiXNyx1dYojzJKAywnI00FFhV4nKVgKbycES/oBFi5UHGGWBFxMQNZqd2J9NwqWclE4IVjiQnIyhzgoTcGsiwnIBH6JiqM7UbAU4k4Jlsy9fjz/WvGIWRJwHwGnQtdRsJRrwSnBkiEOQ4YozjBLAi4kcPo0sGaNM45RsBTuTgmWuCHxC8+dUxxilgRcRkAWobQ63mC0VaRgKaScFCx5UxgIKA4xSwIuIlBermdVhu6qTMFSyDgpWOKKrOJw+7biFLMk4BICkyc7OwSHggWZz7cPLS0txiXhtGDJuBZZqoOJBNxGQJZDGj/eWa98L1j5+fl47rnncFeG7ULf8jI9NbsMJH3woKcjuI8E9BOYPx/46Sf9dkMt+lqw6uvrkZGRgVGjRrlKsGT55MWLQ5uJ30nAWQLSye7UUIbQmvtasESspk+fjldeeQUbN240uAwZMgS5ubkocnjRdVnF4dGj0KbidxJwjsB33wGHDjlnX7ps5HeZmZnp36g5EntQYhAmJSU9ESin+7DMS+LAAfvCfZs2+EkC0RCQ8F3Dh0dzpP3H+PoOy8RbXl6uJfKzaS/aT66VFS0pHmcnAVlkcssWOy1EXzYFS2HlljsscWvTJiA/X3GQWRLQTEDCd7klUbCUlnCTYEkflowqZiIBpwgcOQJkZTllvatdCpbCxE2CJa6tWAEUFipOMksCmgjIeu1tbZqMRWGGgqVAcptgyevk0aMVJ5klAQ0EZJ02t4Wio2ApDe82wRL35s4FJLIuEwnoJCDdEfX1Oi1GtkXBUhi5UbBk1LvMMWQiAV0ESkuBmTN1WYveDgVLYeVGwRIXP/2Uk6KVpmLWRgJjxwKPZ6vZaCX2oilYCjO3CpZcPHIRMZGA3QRu3uz4A2m3nXjKp2Ap1N766C20PmpVtrojK0t76I5S4o6a0wudBCTsnBvvroTB4g2L8XX21zpxPLH1zJNvLvoyLHUYBu8YjNJ7pS7yqsOVmhpg0iTXuUWHehEBiZOpO9ZgtPjmHpuLtK/S/DuXMBwoeSRsDjZjyg9TsPva7nCHOLpN/vqJcDGRgB0Epk8Hbt2yo+T4y7zffB9Ddg7BoeuHDLGSx0InkivvsEL7sNaeX2sI16N29yybIBfTtGlONBdt9nYCtbXui0J+4c4FDN05FFWNVQZ+9mEpV2GoYMmuM7fPIGlbEu48uKMc6VxW7rKqq52zT8u9k4C8iXbTdbXh4gZ8dvQzBNuDT4BTsJ6g6PiiCpZsrWuuw8c5H+NYuQORIxX/JCtvcWbNCrODm0ggTgLSzeCWsX4twRYEcgPYVrytS20oWAqScIIlh7SjHV+e/BKLChYpZziTlTeG0kHKRAJWEJCXOW64u6qorzD6q67dvRa2WhQsBUt3gmUe9kPZD0jbm4YHrc4uui4XF4NVmK3Cz0QIyB27dDM4nfaW7EV6bjoetj3s1hUKloImkmDJ4TfrbxpDH7r7K6AUaVuWd1m2ofVVwfIoWFnpXJXlpdbMwzPxzflvIjrhe8GaMWMGUlNTsWrVKgNWNIIlB7Y9asO0/GnYdHlTRMh2HVBV5Y6/jHbVj+XaT0DuruQPn1Op9mEtRuwegdO3Tkflgu8FSyjdu3cPH3zwgQEsWsEy6W65sgUT8iY4NjpehjjcuGF6w08SiI2ArMgg8QadSOcqz2HYrmGoaYp+YKHvBevChQvo27cvli1bZrRZSkqKEZhCAlREm67UXDE6Cm816B9xJ1MouF5WtC3F40IJXL0KZGaGbtH3feWZlZh/fL7xMisaq8Fg0PhdZmVlcaS7ABswYIDBLRAIoLa2Fo2NjdFwfHJMQ0sDUvak4PCNw0+26foybx5w6ZIua7TTWwhInMGH3fdv21LNprYmY4jQrmu7Yiq/vb3d+F1mZ2f7V7DkUVAESmIRbt++3QAY6yOhSn1F4QrInCcZBqEryUXnhiCXuupLO4kTOH4c+OqrxMuJpQSZnyvzdG/Uxd+H4ftHQhV4ooIl5Z2sOGl0JMocKF1J1n4/dUqXNdrxOoFhw4CWFn21kHm5Mj9X5ukmkihYCj0rBEuKrG6sNjoUZS6UjiQRdiRgABMJRCKwdy8gkZx1JBmyIEIl83KtSBQshaJVgiXFyhyo2UdmY/3F9YoVe7IS7NKhiez2VIilWk5AojgPHgzIp91J5t/KPNyzlWctM0XBUlBaKVhm0duLtxsjeGWOlN1Jgl7K3RYTCYQjsH49sH9/uD3WbpN5tzL/VubhWpkoWApNOwRLTFy/f93ocPzp3k+KRWuzZ88CK1daWyZL6x0E5OXMiBH210Xm28q8WztePFGwlPazS7DEjMyRGr1vNPb8uEexam1WxmW5LTyTtTVkafEQmD8fuHIlnjOjO6extdGYZyvzbe1KFCyFrJ2CZZqSDshZh2fBroUBZeT7Z5+Z1vhJAoAszhcI2EdC5tXKqqAyz9bORMFS6OoQLDFZeLsQw3cNh8ylsiPJelnXr9tRMsv0IgGZ4GxXYAmZTyvzamV+rd2JgqUQ1iVYYlbGaY3cPRInbp5QvEg8K4+EaWmJl8MSvE/g8mXg88+tr4dEl5J5tDKfVleiYCmkdQqWmJaOSZlTtbxwueJJ4tl164C8vMTLYQneJiDj89osvvm53XDbeASUebQ6EwVLoa1bsEzzeT/lGR2W0nFpZZIxN1ZfrFb6x7LsJbB1q/XDGGS+rMyblfmzuhMFSyHulGCJG+V15cbQh5LaEsWr+LMnTgDLrb95i98hnqmNgEy9sXqOqcyTlfmyTiUKlkLeScESV8yFAb8v+l7xLP5sero71uuOvwY8Mx4CsnTMtfBLo8dcnBkb0InVSEKdpWCF0gDgtGCZ7qy7sA4ZBzM6hTgy98X6efu2eyKixOo7j4+PgLwhzsiI71z1LDU2oLpfZ56CpdB2i2CJW0XVRUbHZuWDxBfcXrIEOOaOKGUKcWbtICB9l01NiZccLjZg4qXGXwIFS2HnJsES1+pb6vFJzic4Wn5U8TS2rMwvHDSIHfCxUfPm0TIJPicnMd9l3qtEsAkXGzCxkhM7m4Kl8HObYJnuSUfnnKNzzGxcnwUF+hdti8tRnhQ3AVkoN9H5gmX3yzBw60A4HRUqHATfC1ZOTg7WrFljBKIQQG4VLPEt/3o+UvekJhQTcdw4Z0M6hbsIuc06AtJvlUhQEokNKPNde4oNaJ23sZfke8GSNaILCgrw97//3aDnZsESB6saqzB051BcrLoYe2vL6Pr7QEpKXKfyJJcTKCwEFsUZmFzmtcr81mhiAzqJwfeCJfAlQk5ycrLRDm4XLHFShj58mv8pvrsU37KRu3cDm5wLp+jk9d5rbbe2dizMF89aaLHGBnQSou8FSwJRfPTRR6ip6YiNJgEpcnNzUVRU5GS7RGV765WtCOQG4oqJKHdZTsWji6pyPCgmAhI56dy5mE4xDpbVQGONDRi7lcTPaGlpMX6XmZmZ/o2a8+jRI7z00ktYvHgxdu7caVD1wh1WaPNfvXvVGB1fUV8Rujnid9FnxjOMiMkTB1y4EF/05lhjA7oBhq/vsCTW2alTp4x/58+fN9rDa4IlTksHqbyCzi3JjemaysoCfrBvrbWYfOHB8RGQR0AJWh7LmCuJDZi2Nw37SzWslRxftbo9y9eCFY6KFwXLrIes+DD90PSYFgaUuWayuBuTNwnMmQNIZ3u0yYrYgNHasuM4CpZC1cuCJVU5deuUERPx3sN7Ss3CZ2Xajgx1YPIeAQk1P3589H5bFRsweovWH0nBUph6XbCkOjVNNUZH6rnK6HphZQmSHTsUEMy6moC8FRwwAGiOIi6p1bEBnQRDwVLo9wbBkiqZCwNKx2o0KTUVuBfdTVk0xfEYmwnImv1nzkQ2YkdswMhW7TuCgqWw7S2CZVZr59WdRnw46WjtKckQB4lpyOR+ArLG2YIFkf20KzZgZMv2HUHBUtj2NsGS6t2ou2EMfZAO156SLKe81pqI4j2Z4b4ECDx4AAwdGjlYrp2xARNwP+FTKVgKwt4oWFLF5mAzpvwwBdLx2lOaMcPe2HU92ea+yASGD+95LuiD1gfGkIWDZQcjF+bBIyhYSqP1VsEyqykxEUW4uouJKMvqyjI0EiWYyV0EvvkGyM7u3qfL1ZeNVRbsjg3YvQf276FgKYx7u2BJdWU6RtK2JEiHbLgks/3tDLoZzia39UxAQnVJbMHuksQGlPmlOmIDdueDju0ULIWyHwRLqlzXXGd0xkvHbLi0cSPwXXxzq8MVx20JEJChCwMHhr/rdSI2YAJVSfhUCpaC0C+CJdWWoQ9fnvwS0kEbLo0ZY10Qg3Dlc1t0BGTOZ7hgErcabjkSGzA6r+05ioKlcPWTYJlVlw5amVsmHbahSf6yJyUBsoolkzMEuuu3cjI2oDMkOqxSsBT6fhQsQSAdtR9u+xDScRuaysutj20XWj6/d0/g4kVg0qTO++Wu2OnYgJ090pujYCm8/SpYgsFcGFA6cENTfj4wf37oFn63m8Ddu12DhkhswJG7R6KgosBu864tn4KlNI2fBctEseXKFkw8MLHTwoASJmzPHvMIftpJoK0NkAVwRbTMZMYGrG6sNjf58pOCpTQ7BasDSHFNsdGhKx27ZpJVHTywEKvprmc/ZfiCPA6ayW2xAU2/nPikYCnUKVhPgTS0NBhL1ZgLvclffgnQGfqX/+nR/GYFgWXLgMeL38KMDbi9eLsVRfeKMnwvWGvXrsVrr72GO3c6BlFSsLpe1xITUTp6pcNXFvuTx5VoljXpWhK39ERAhEoES5KbYwP2VAe79/lesATw+PHjcetWx6MPBSv8JXey4qRxtyUdv2VlHW8O44nQEr50bj19Gpg6tYOD22MDOtlaFCxFsFJSUlBSUmKE/nKyYdxoWzp8JSbi+TvncfYsMHGiG730nk/FxYCsRxZ85I3YgE4QDgaDxu8yKyvLv1FzTPChd1iBQAC1tbVo5GhJE0+nz2B7EDMPz8Tqs6shy9HMndtpNzMxEpB5m/KIfae+FsN3DcfpW6djLMEfh0vAGPldSuBjuctyIj3jhFHV5sGDBzFixAhMnToVDQ0Nrg5Vr/ruZF46giVSz/dbW7BwoZOeeNe2dJv27w8cL/NGbEA3kOYjodIK7MNSgPSQvX7/urEw4PzVPz3pLO7hcO4KIVBV1bEm+xeHV2D+8fnGC42Q3fzaDQEKlgKGgqUAiZCVmIhj9o1ByqI9WL48wsHcbRCQILb9BjZh2LY05P2URyoxEKBgKbAoWAqQKLOyMOD/zJ+FJUsfRXmGPw+rrgbeHlSKDzYPNpau9ieF+GtNwVLYUbAUIDFkC28X4j/mDsfshYzMGg6b9Fm9NnIXxuRMMZasDncMt/VMgIKl8KFgKUBizMrCgH0WjUTy1BMxntm7Dy8qDuKF9AysPPlN766ozbWjYCmAKVgKkDiyMiJ+xLr5eHXscgSDcRTQy07ZebASvxybhFM3z/aymumvDgVLYU7BUoAkkF39wwE8GxiFO7X+XQFw1jfH8PuMj3H/YV0CJHmqSYCCZZJ4/EnBUoAkmD1TUo5/HjUYhy79mGBJ3jv9bzMXof/CLzlkwcKmo2ApMClYChALsrUNTfjNuADmbsm1oDT3F1Fb34zfjBmDOVt7jgHp/pq4z0MKltImFCwFiIXZ/5uzEn1mfIbgo3YLS3VXUXtPX8Uv0ofgRHGZuxzrJd5QsJSGpGApQCzOrtl7Gr8YNRxnSyosLtn54gYv/gZ/njIVjc2tzjvTSz2gYCkNS8FSgNiQLa++h+cnjEDqqm9tKF1/kcXlNXg2PQ2zN3HUut30KVgKYQqWAsTGbPqaTfjXMcNwsey2jVbsLfrjZRvw3NiPUVJx315DLN0gQMFSLgQKlgLE5uy1m7X4t3FpeG/uUrS2eWdazw/nSvCr8cmYsn6rzYRYfCgBClYoDYDLyyg8dGWX5Rw1+rbmbDqoy2Rcdsqr6vHylGl4fXom7jZ0DjwbV4E8KSYCFCwFF++wFCAas+3tQGDlFvwyfSgWbneXcN2qacAb0xbiT+On4+y1Ko1UaCqUAAUrlAbvsBQazmTl0XDE0m/xq9EjMW71TrS2OTcM4sy12/j3jIl4YWIqCoqehjxzhgyt+l6wLl++jA0bNqC1teNVNO+w3POjkDmJszbtxS8Cw4zHsKMX9AyFqHvQivSV2/BsIIDXp83GpbKOiEruIeNfT3wtWDU1NRg0aBCOHTuGcRIlFBIefJD2q+HKlSu0GYHApeuVeGfOF3hxwji8MWMeNh0+awRtiHAaomV7594DjFm1Hb8dPQEvjJmIz7MPoy0Y351dtDYj+R7LfrEp657rTDdu3NAe+2Dz5s2Qf04kx9d037NnDzZu3GjUPSkpyfh89913tbPYv38/bcZA4FpFDYYtWYdfj03Br8d+gj9P+RRjVm+FjDJvePiwU0nh2BZdr8aKPcfx/vyVeHFsBv4wLgOvTcvE6twCtAUTf1MZzmYnp2zI5OXlaReswsJCIzCEDdXptkiJIyr/nEiOC9bevXuxfv16o+4ffvih8Tl06FC88847kHx6erqWf4MHD9ZiJ7Q+vcnmoGEj8B///S7+5c//iZ///kX84+9+i5/97gX87Pcv4B9+/azx+fMXX8DP//BbY98//ekl/Oa/3sT/9k9GIGB9G/cmtqHXjPp92LBhSE1N1XLtpqWlGb/LV199FRcuXHBCr+C4YNXV1WHAgAHYsmULZs6c6QgEGiUBEvAGAccFSzDdvHkThw4dCns7fe/ePVy6dEkbzYqKCiPmmvQN6ErV1dWGTQkeqys1Nzfj9OnTuCNrBtucSktLcfz4cZutdC5eooifOKFvxVXpu5LHM3ksNF8edfbIntyZM2cgj78Plcdwe6w9LfXatWtarp2nFju+uUKwVKdC81OmTMHAgQNDN9n6/ciRI9i9ezdef/11iFjqSDt27EBOTg7efPNNlJeX6zCJU6dOITk5+Un/oV1GpT5DhgzB6tWrsWLFCrvMdCl30aJFRht22WHThvr6eqxatQrr1q2DPI7qSt9//z02bdqEfv366TKJ+/fv4+WXX4bY1p1cLVjSEPKXWadgSfpyKFEAAAJVSURBVAMsXbrU+DG3tbVpbQ/pi7h9W9+cPhFm84WHXRX9+uuvceDAAaN43e1o9onaVbdw5VZWViIlJSXcLlu2Xb9+HbNnz8aCBQtsKT9coZMnTzaGIflasCTqs4y/kn8TJ05ES0sL3nrrLeTm5kI6+e7evRuOXULb5K+waVM+pT9N0sWLFzFy5EhUSaRNi5P8BQ61WVbWsWaT3IHIj9uOJKIUalNu5yXpECypV35+vmFPt4Dotid3WR988IHWt3aNjY1Gd0ogELDj0ulSpjyBiCDL7zUjI6PLfrs3uPYOKxgMGo8t8ujyxhtvQC4GHen8+fMoLi42xoLp6lOSO0kZeyZ11VVPua3/4osvMGvWLFt/YDI2SX5M0reTmZmpowkNG2L3r3/9q7a3WdIn2KdPH8jjfVFRkZZ6ik3pN5N/7733nhabcuMg1+m8efMwZ84cLTZDjbhWsEKd1NnpLo+gy5cvN+6yQn2w87u8IpY7SflXW6snnqB09Js25THGznT48GGjb0f+COlKJ0+eNOondwQ6knR6mzwLCgp0mIR0WXz33XfIysrS1t9qVkyuHx0vbEx75qcnBMt0lp8kQAL+JkDB8nf7s/Yk4CkCFCxPNRedJQF/E6Bg+bv9WXsS8BQBCpanmovOkoC/CVCw/N3+rD0JeIoABctTzUVnScDfBChY/m5/1p4EPEWAguWp5qKzJOBvAhQsf7c/a08CniJAwfJUc9FZEvA3AQqWv9uftScBTxGgYHmquegsCfibAAXL3+3P2pOApwhQsDzVXHSWBPxN4P8BoAlIAsWP3jgAAAAASUVORK5CYII="
    }
   },
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](attachment:image.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a custom loss function\n",
    "# Define Huber loss function\n",
    "\n",
    "def huber_fn(y_true, y_pred):\n",
    "    error = y_true - y_pred\n",
    "    is_small_error = tf.abs(error) < 1\n",
    "    squared_loss = tf.square(error) / 2\n",
    "    linear_loss  = tf.abs(error) - 0.5\n",
    "    return tf.where(is_small_error, squared_loss, linear_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build the network.\n",
    "# Output layer just contains 1 neuron since we have to predict only one value\n",
    "\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=X_train.shape[1:]),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify the loss function and the optimizer to use.\n",
    "# Here we are using custom loss function\n",
    "# Measure MAE during training and evaluation\n",
    "\n",
    "model.compile(loss=huber_fn, optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 116us/sample - loss: 0.6180 - mae: 0.9793 - val_loss: 0.2435 - val_mae: 0.5304\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 62us/sample - loss: 0.2080 - mae: 0.4988 - val_loss: 0.1915 - val_mae: 0.4737\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbccc1387b8>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the model\n",
    "\n",
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saving/Loading Models with Custom Objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the model\n",
    "\n",
    "model.save(\"my_model_with_a_custom_loss.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.engine.sequential.Sequential at 0x7fbcc455cdd8>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the model\n",
    "# Specify the custom loss function while loading the model\n",
    "\n",
    "loaded_model = keras.models.load_model(\"my_model_with_a_custom_loss.h5\", custom_objects={\"huber_fn\": huber_fn})\n",
    "loaded_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 72us/sample - loss: 0.2006 - mae: 0.4893 - val_loss: 0.1926 - val_mae: 0.4771\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 71us/sample - loss: 0.1970 - mae: 0.4841 - val_loss: 0.1840 - val_mae: 0.4633\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbcc439b860>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When you save the model, the threshold will not be saved. This means that you will have to specify the threshold value when loading the model. You can solve this by creating a subclass of the keras.losses.Loss class, and then implementing its get_config() method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberLoss(keras.losses.Loss):\n",
    "    def __init__(self, threshold=1.0, **kwargs):\n",
    "        self.threshold = threshold\n",
    "        super().__init__(**kwargs)\n",
    "    def call(self, y_true, y_pred):\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < self.threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss  = self.threshold * tf.abs(error) - self.threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_shape = X_train.shape[1:]\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=HuberLoss(2.), optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 115us/sample - loss: 0.2095 - mae: 0.4750 - val_loss: 0.1956 - val_mae: 0.4635\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 64us/sample - loss: 0.2069 - mae: 0.4727 - val_loss: 0.2107 - val_mae: 0.4594\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbcc419ae10>"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_loss_class.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 64us/sample - loss: 0.2036 - mae: 0.4677 - val_loss: 0.2272 - val_mae: 0.4775\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 63us/sample - loss: 0.2038 - mae: 0.4682 - val_loss: 0.1924 - val_mae: 0.4482\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbcc40ce6a0>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.loss.threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Custom Activation Functions, Initializers, Regularizers, and Constraints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clearing the session for any left over residues\n",
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining a custom Activation Function\n",
    "def my_softplus(z): # return value is just tf.nn.softplus(z)\n",
    "    return tf.math.log(tf.exp(z) + 1.0)\n",
    "\n",
    "# Defining a custom Initializer\n",
    "def my_glorot_initializer(shape, dtype=tf.float32):\n",
    "    stddev = tf.sqrt(2. / (shape[0] + shape[1]))\n",
    "    return tf.random.normal(shape, stddev=stddev, dtype=dtype)\n",
    "\n",
    "# Defining a custom Regularizer\n",
    "def my_l1_regularizer(weights):\n",
    "    return tf.reduce_sum(tf.abs(0.01 * weights))\n",
    "\n",
    "# Defining a custom Constraint\n",
    "def my_positive_weights(weights): # return value is just tf.nn.relu(weights)\n",
    "    return tf.where(weights < 0., tf.zeros_like(weights), weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining a layer in Keras with the custom activation function,\n",
    "# initializer, regularizer, and constraint defined above\n",
    "\n",
    "layer = keras.layers.Dense(1, activation=my_softplus,\n",
    "                           kernel_initializer=my_glorot_initializer,\n",
    "                           kernel_regularizer=my_l1_regularizer,\n",
    "                           kernel_constraint=my_positive_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clearing the session for any left over residues\n",
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using the layer above layer in a Sequential model\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1, activation=my_softplus,\n",
    "                       kernel_regularizer=my_l1_regularizer,\n",
    "                       kernel_constraint=my_positive_weights,\n",
    "                       kernel_initializer=my_glorot_initializer),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_2 (Dense)              (None, 30)                270       \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 301\n",
      "Trainable params: 301\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary() # create a model summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compiling the above model\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 2s 131us/sample - loss: 1.8303 - mae: 0.9735 - val_loss: inf - val_mae: inf\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 63us/sample - loss: 0.6573 - mae: 0.5451 - val_loss: inf - val_mae: inf\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbcc4063f28>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Trainging the model\n",
    "model.fit(X_train_scaled, y_train, epochs=2, validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving the trained model\n",
    "model.save(\"my_model_with_many_custom_parts.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading the saved model with the custom objects\n",
    "model = keras.models.load_model(\n",
    "    \"my_model_with_many_custom_parts.h5\",\n",
    "    custom_objects={\n",
    "       \"my_l1_regularizer\": my_l1_regularizer,\n",
    "       \"my_positive_weights\": my_positive_weights,\n",
    "       \"my_glorot_initializer\": my_glorot_initializer,\n",
    "       \"my_softplus\": my_softplus,\n",
    "    })"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Custom Function with Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Simple class for ℓ1 regularization that saves its factor hyperparameter\n",
    "class MyL1Regularizer(keras.regularizers.Regularizer):\n",
    "    def __init__(self, factor):\n",
    "        self.factor = factor\n",
    "    def __call__(self, weights):\n",
    "        return tf.reduce_sum(tf.abs(self.factor * weights))\n",
    "    def get_config(self):\n",
    "        return {\"factor\": self.factor}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining a Sequential model with custom objects\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1, activation=my_softplus,\n",
    "                       kernel_regularizer=MyL1Regularizer(0.01),\n",
    "                       kernel_constraint=my_positive_weights,\n",
    "                       kernel_initializer=my_glorot_initializer),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compiling the above model\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 119us/sample - loss: 1.4486 - mae: 0.8727 - val_loss: 2.4208 - val_mae: 0.5681\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 64us/sample - loss: 0.5848 - mae: 0.5260 - val_loss: 1.6040 - val_mae: 0.5122\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbc8c6288d0>"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Training the model\n",
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_many_custom_parts.h5\") # Save the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading the saved model with custom objects\n",
    "model = keras.models.load_model(\n",
    "    \"my_model_with_many_custom_parts.h5\",\n",
    "    custom_objects={\n",
    "       \"MyL1Regularizer\": MyL1Regularizer,\n",
    "       \"my_positive_weights\": my_positive_weights,\n",
    "       \"my_glorot_initializer\": my_glorot_initializer,\n",
    "       \"my_softplus\": my_softplus,\n",
    "    })"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Custom Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining a Huber loss function we will use as\n",
    "# a custom metric function\n",
    "def create_huber(threshold=1.0):\n",
    "    def huber_fn(y_true, y_pred):\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss  = threshold * tf.abs(error) - threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    return huber_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# Creating a Sequential model\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compiling the above model with the Huber loss function\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[create_huber(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 50us/sample - loss: 0.5015 - huber_fn: 0.2380\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 49us/sample - loss: 0.4771 - huber_fn: 0.2308\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbc8c16d4e0>"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Warning**: if you use the same function as the loss and a metric, you may be surprised to see different results. This is generally just due to floating point precision errors: even though the mathematical equations are equivalent, the operations are not run in the same order, which can lead to small differences. Moreover, when using sample weights, there's more than just precision errors:\n",
    "* the loss since the start of the epoch is the mean of all batch losses seen so far. Each batch loss is the sum of the weighted instance losses divided by the _batch size_ (not the sum of weights, so the batch loss is _not_ the weighted mean of the losses).\n",
    "* the metric since the start of the epoch is equal to the sum of weighted instance losses divided by sum of all weights seen so far. In other words, it is the weighted mean of all the instance losses. Not the same thing.\n",
    "\n",
    "If you do the math, you will find that loss = metric * mean of sample weights (plus some floating point precision error)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=create_huber(2.0), optimizer=\"nadam\", metrics=[create_huber(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Train on 11610 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 106us/sample - loss: 0.1174 - huber_fn: 0.2375\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 52us/sample - loss: 0.1137 - huber_fn: 0.2306\n"
     ]
    }
   ],
   "source": [
    "sample_weight = np.random.rand(len(y_train))\n",
    "history = model.fit(X_train_scaled, y_train, epochs=2, sample_weight=sample_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.11744698982085779, 0.11787284694663312)"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history.history[\"loss\"][0], history.history[\"huber_fn\"][0] * sample_weight.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Streaming metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.8>"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The the keras.metrics.Precision class can keep track of\n",
    "# the number of true positives and the number of false positives\n",
    "# and that can compute their ratio when requested\n",
    "precision = keras.metrics.Precision()\n",
    "precision([0, 1, 1, 1, 0, 1, 0, 1], [1, 1, 0, 1, 0, 1, 0, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.5>"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision([0, 1, 0, 0, 1, 0, 1, 1], [1, 0, 1, 1, 0, 0, 0, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.5>"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'true_positives:0' shape=(1,) dtype=float32, numpy=array([4.], dtype=float32)>,\n",
       " <tf.Variable 'false_positives:0' shape=(1,) dtype=float32, numpy=array([4.], dtype=float32)>]"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "precision.reset_states()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.5>"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision = keras.metrics.Precision()\n",
    "precision([0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1], [1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0])\n",
    "precision.result()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating a streaming metric:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberMetric(keras.metrics.Metric):\n",
    "    def __init__(self, threshold=1.0, **kwargs):\n",
    "        super().__init__(**kwargs) # handles base args (e.g., dtype)\n",
    "        self.threshold = threshold\n",
    "        #self.huber_fn = create_huber(threshold) # TODO: investigate why this fails\n",
    "        self.total = self.add_weight(\"total\", initializer=\"zeros\")\n",
    "        self.count = self.add_weight(\"count\", initializer=\"zeros\")\n",
    "    def huber_fn(self, y_true, y_pred): # workaround\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < self.threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss  = self.threshold * tf.abs(error) - self.threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    def update_state(self, y_true, y_pred, sample_weight=None):\n",
    "        metric = self.huber_fn(y_true, y_pred)\n",
    "        self.total.assign_add(tf.reduce_sum(metric))\n",
    "        self.count.assign_add(tf.cast(tf.size(y_true), tf.float32))\n",
    "    def result(self):\n",
    "        return self.total / self.count\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Warning**: when running the following cell, if you get autograph warnings such as `WARNING:tensorflow:AutoGraph could not transform [...] and will run it as-is`, then please install version 0.2.2 of the gast library (e.g., by running `!pip install gast==0.2.2`), then restart the kernel and run this notebook again from the beginning (see [autograph issue #1](https://github.com/tensorflow/autograph/issues/1) for more details):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=14.0>"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = HuberMetric(2.)\n",
    "\n",
    "# total = 2 * |10 - 2| - 2²/2 = 14\n",
    "# count = 1\n",
    "# result = 14 / 1 = 14\n",
    "m(tf.constant([[2.]]), tf.constant([[10.]])) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=7.0>"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# total = total + (|1 - 0|² / 2) + (2 * |9.25 - 5| - 2² / 2) = 14 + 7 = 21\n",
    "# count = count + 2 = 3\n",
    "# result = total / count = 21 / 3 = 7\n",
    "m(tf.constant([[0.], [5.]]), tf.constant([[1.], [9.25]]))\n",
    "\n",
    "m.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'total:0' shape=() dtype=float32, numpy=21.0>,\n",
       " <tf.Variable 'count:0' shape=() dtype=float32, numpy=3.0>]"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'total:0' shape=() dtype=float32, numpy=0.0>,\n",
       " <tf.Variable 'count:0' shape=() dtype=float32, numpy=0.0>]"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.reset_states()\n",
    "m.variables"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's check that the `HuberMetric` class works well:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=create_huber(2.0), optimizer=\"nadam\", metrics=[HuberMetric(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 97us/sample - loss: 0.2265 - huber_metric_1: 0.2265\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 50us/sample - loss: 0.2220 - huber_metric_1: 0.2220\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbc8c294c50>"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled.astype(np.float32), y_train.astype(np.float32), epochs=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_metric.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 51us/sample - loss: 0.2173 - huber_metric_1: 0.2173\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 49us/sample - loss: 0.2148 - huber_metric_1: 0.2148\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbc8c08ef98>"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled.astype(np.float32), y_train.astype(np.float32), epochs=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Warning**: In TF 2.2, tf.keras adds an extra first metric in `model.metrics` at position 0 (see [TF issue #38150](https://github.com/tensorflow/tensorflow/issues/38150)). This forces us to use `model.metrics[-1]` rather than `model.metrics[0]` to access the `HuberMetric`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.metrics[-1].threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looks like it works fine! More simply, we could have created the class like this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberMetric(keras.metrics.Mean):\n",
    "    def __init__(self, threshold=1.0, name='HuberMetric', dtype=None):\n",
    "        self.threshold = threshold\n",
    "        self.huber_fn = create_huber(threshold)\n",
    "        super().__init__(name=name, dtype=dtype)\n",
    "    def update_state(self, y_true, y_pred, sample_weight=None):\n",
    "        metric = self.huber_fn(y_true, y_pred)\n",
    "        super(HuberMetric, self).update_state(metric, sample_weight)\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This class handles shapes better, and it also supports sample weights."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=keras.losses.Huber(2.0), optimizer=\"nadam\", weighted_metrics=[HuberMetric(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Train on 11610 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 120us/sample - loss: 0.4330 - HuberMetric: 0.8725\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 52us/sample - loss: 0.1286 - HuberMetric: 0.2592\n"
     ]
    }
   ],
   "source": [
    "sample_weight = np.random.rand(len(y_train))\n",
    "history = model.fit(X_train_scaled.astype(np.float32), y_train.astype(np.float32),\n",
    "                    epochs=2, sample_weight=sample_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.43299613758652095, 0.4329960495905965)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history.history[\"loss\"][0], history.history[\"HuberMetric\"][0] * sample_weight.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_metric_v2.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 99us/sample - loss: 0.2363 - HuberMetric: 0.2363\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 51us/sample - loss: 0.2274 - HuberMetric: 0.2274\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbc75e12d30>"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled.astype(np.float32), y_train.astype(np.float32), epochs=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Warning**: In TF 2.2, tf.keras adds an extra first metric in `model.metrics` at position 0 (see [TF issue #38150](https://github.com/tensorflow/tensorflow/issues/38150)). This forces us to use `model.metrics[-1]` rather than `model.metrics[0]` to access the `HuberMetric`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.metrics[-1].threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Custom Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a custom layer without any weights\n",
    "exponential_layer = keras.layers.Lambda(lambda x: tf.exp(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=float32, numpy=array([0.36787945, 1.        , 2.7182817 ], dtype=float32)>"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exponential_layer([-1., 0., 1.])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adding an exponential layer at the output of a regression model can be useful if the values to predict are positive and with very different scales (e.g., 0.001, 10., 10000):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/5\n",
      "11610/11610 [==============================] - 1s 114us/sample - loss: 66233.5680 - val_loss: 18662058550442885943476092928.0000\n",
      "Epoch 2/5\n",
      "11610/11610 [==============================] - 1s 59us/sample - loss: 32.4438 - val_loss: 17632522328030297767505035264.0000\n",
      "Epoch 3/5\n",
      "11610/11610 [==============================] - 1s 57us/sample - loss: 32.0129 - val_loss: 23047029300062756774573244416.0000\n",
      "Epoch 4/5\n",
      "11610/11610 [==============================] - 1s 70us/sample - loss: 31.5074 - val_loss: 224675980896447727608998658048.0000\n",
      "Epoch 5/5\n",
      "11610/11610 [==============================] - 1s 59us/sample - loss: 30.8829 - val_loss: 13163095458361068364115214336.0000\n",
      "5160/5160 [==============================] - 0s 29us/sample - loss: 0.5564\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.5563613043274991"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"relu\", input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "    exponential_layer\n",
    "])\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "model.fit(X_train_scaled, y_train, epochs=5,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11610, 8)"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_scaled.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8,)"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a custom stateful layer (i.e., a layer with weights)\n",
    "class MyDense(keras.layers.Layer):\n",
    "    def __init__(self, units, activation=None, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.units = units\n",
    "        self.activation = keras.activations.get(activation)\n",
    "\n",
    "    def build(self, batch_input_shape):\n",
    "        self.kernel = self.add_weight(\n",
    "            name=\"kernel\", shape=[batch_input_shape[-1], self.units],\n",
    "            initializer=\"glorot_normal\")\n",
    "        self.bias = self.add_weight(\n",
    "            name=\"bias\", shape=[self.units], initializer=\"zeros\")\n",
    "        super().build(batch_input_shape) # must be at the end\n",
    "\n",
    "    def call(self, X):\n",
    "        return self.activation(X @ self.kernel + self.bias)\n",
    "\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        return tf.TensorShape(batch_input_shape.as_list()[:-1] + [self.units])\n",
    "\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"units\": self.units,\n",
    "                \"activation\": keras.activations.serialize(self.activation)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    MyDense(30, activation=\"relu\", input_shape=input_shape),\n",
    "    MyDense(1)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 2s 144us/sample - loss: 2.2020 - val_loss: 2.7150\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 68us/sample - loss: 0.6063 - val_loss: 0.5397\n",
      "5160/5160 [==============================] - 0s 29us/sample - loss: 0.5004\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.5004428601080133"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "my_dense_4 (MyDense)         (None, 30)                270       \n",
      "_________________________________________________________________\n",
      "my_dense_5 (MyDense)         (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 301\n",
      "Trainable params: 301\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_layer.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\"my_model_with_a_custom_layer.h5\",\n",
    "                                custom_objects={\"MyDense\": MyDense})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a layer with multiple inputs\n",
    "class MyMultiLayer(keras.layers.Layer):\n",
    "    def call(self, X):\n",
    "        X1, X2 = X\n",
    "        return X1 + X2, X1 * X2\n",
    "\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        batch_input_shape1, batch_input_shape2 = batch_input_shape\n",
    "        return [batch_input_shape1, batch_input_shape2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs1 = keras.layers.Input(shape=[2])\n",
    "inputs2 = keras.layers.Input(shape=[2])\n",
    "outputs1, outputs2 = MyMultiLayer()((inputs1, inputs2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's create a layer with a different behavior during training and testing:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a layer that adds Gaussian noise\n",
    "# during training (for regularization)\n",
    "# but does nothing during testing\n",
    "class MyGaussianNoise(keras.layers.Layer):\n",
    "    def __init__(self, stddev, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.stddev = stddev\n",
    "\n",
    "    def call(self, X, training=None):\n",
    "        if training:\n",
    "            noise = tf.random.normal(tf.shape(X), stddev=self.stddev)\n",
    "            return X + noise\n",
    "        else:\n",
    "            return X\n",
    "\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        return batch_input_shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 103us/sample - loss: 0.4517 - val_loss: 0.4844\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 57us/sample - loss: 0.4073 - val_loss: 0.3831\n",
      "5160/5160 [==============================] - 0s 28us/sample - loss: 0.3900\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.3899582788001659"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Custom Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_new_scaled = X_test_scaled"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating a Custom Model by Subclass the keras.Model class, create layers and variables in the constructor, and implement the call() method to do whatever you want the model to do."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Layer containing another layer\n",
    "class ResidualBlock(keras.layers.Layer):\n",
    "    def __init__(self, n_layers, n_neurons, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden = [keras.layers.Dense(n_neurons, activation=\"elu\",\n",
    "                                          kernel_initializer=\"he_normal\")\n",
    "                       for _ in range(n_layers)]\n",
    "\n",
    "    def call(self, inputs):\n",
    "        Z = inputs\n",
    "        for layer in self.hidden:\n",
    "            Z = layer(Z)\n",
    "        return inputs + Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# Using the Subclassing API to define the model\n",
    "class ResidualRegressor(keras.models.Model):\n",
    "    def __init__(self, output_dim, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden1 = keras.layers.Dense(30, activation=\"elu\",\n",
    "                                          kernel_initializer=\"he_normal\")\n",
    "        self.block1 = ResidualBlock(2, 30)\n",
    "        self.block2 = ResidualBlock(2, 30)\n",
    "        self.out = keras.layers.Dense(output_dim)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        Z = self.hidden1(inputs)\n",
    "        for _ in range(1 + 3):\n",
    "            Z = self.block1(Z)\n",
    "        Z = self.block2(Z)\n",
    "        return self.out(Z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples\n",
      "Epoch 1/5\n",
      "11610/11610 [==============================] - 2s 175us/sample - loss: 11.4379\n",
      "Epoch 2/5\n",
      "11610/11610 [==============================] - 1s 78us/sample - loss: 1.7623\n",
      "Epoch 3/5\n",
      "11610/11610 [==============================] - 1s 77us/sample - loss: 1.4745\n",
      "Epoch 4/5\n",
      "11610/11610 [==============================] - 1s 77us/sample - loss: 0.6445\n",
      "Epoch 5/5\n",
      "11610/11610 [==============================] - 1s 77us/sample - loss: 0.5185\n",
      "5160/5160 [==============================] - 0s 56us/sample - loss: 0.6045\n"
     ]
    }
   ],
   "source": [
    "model = ResidualRegressor(1)\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=5)\n",
    "score = model.evaluate(X_test_scaled, y_test)\n",
    "y_pred = model.predict(X_new_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1786: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "INFO:tensorflow:Assets written to: my_custom_model.ckpt/assets\n"
     ]
    }
   ],
   "source": [
    "model.save(\"my_custom_model.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\"my_custom_model.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples\n",
      "Epoch 1/5\n",
      "11610/11610 [==============================] - 2s 214us/sample - loss: 0.6824\n",
      "Epoch 2/5\n",
      "11610/11610 [==============================] - 1s 81us/sample - loss: 0.4910\n",
      "Epoch 3/5\n",
      "11610/11610 [==============================] - 1s 80us/sample - loss: 0.6513\n",
      "Epoch 4/5\n",
      "11610/11610 [==============================] - 1s 80us/sample - loss: 0.5045\n",
      "Epoch 5/5\n",
      "11610/11610 [==============================] - 1s 80us/sample - loss: 0.6645\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train_scaled, y_train, epochs=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We could have defined the model using the sequential API instead:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "block1 = ResidualBlock(2, 30)\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"elu\", kernel_initializer=\"he_normal\"),\n",
    "    block1, block1, block1, block1,\n",
    "    ResidualBlock(2, 30),\n",
    "    keras.layers.Dense(1)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples\n",
      "Epoch 1/5\n",
      "11610/11610 [==============================] - 2s 172us/sample - loss: 0.3889\n",
      "Epoch 2/5\n",
      "11610/11610 [==============================] - 1s 78us/sample - loss: 0.3642\n",
      "Epoch 3/5\n",
      "11610/11610 [==============================] - 1s 77us/sample - loss: 0.3502\n",
      "Epoch 4/5\n",
      "11610/11610 [==============================] - 1s 76us/sample - loss: 0.3414\n",
      "Epoch 5/5\n",
      "11610/11610 [==============================] - 1s 75us/sample - loss: 0.3436\n",
      "5160/5160 [==============================] - 0s 49us/sample - loss: 0.3461\n"
     ]
    }
   ],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=5)\n",
    "\n",
    "score = model.evaluate(X_test_scaled, y_test)\n",
    "y_pred = model.predict(X_new_scaled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Losses and Metrics Based on Model Internals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom model with a custom reconstruction loss\n",
    "class ReconstructingRegressor(keras.models.Model):\n",
    "    def __init__(self, output_dim, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden = [keras.layers.Dense(30, activation=\"selu\",\n",
    "                                          kernel_initializer=\"lecun_normal\")\n",
    "                       for _ in range(5)]\n",
    "        self.out = keras.layers.Dense(output_dim)\n",
    "        # TODO: check https://github.com/tensorflow/tensorflow/issues/26260\n",
    "        #self.reconstruction_mean = keras.metrics.Mean(name=\"reconstruction_error\")\n",
    "\n",
    "    def build(self, batch_input_shape):\n",
    "        n_inputs = batch_input_shape[-1]\n",
    "        self.reconstruct = keras.layers.Dense(n_inputs)\n",
    "        super().build(batch_input_shape)\n",
    "\n",
    "    def call(self, inputs, training=None):\n",
    "        Z = inputs\n",
    "        for layer in self.hidden:\n",
    "            Z = layer(Z)\n",
    "        reconstruction = self.reconstruct(Z)\n",
    "        recon_loss = tf.reduce_mean(tf.square(reconstruction - inputs))\n",
    "        self.add_loss(0.05 * recon_loss)\n",
    "        #if training:\n",
    "        #    result = self.reconstruction_mean(recon_loss)\n",
    "        #    self.add_metric(result)\n",
    "        return self.out(Z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 2s 172us/sample - loss: 0.7648\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 74us/sample - loss: 0.4110\n"
     ]
    }
   ],
   "source": [
    "model = ReconstructingRegressor(1)\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=2)\n",
    "y_pred = model.predict(X_test_scaled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Computing Gradients with Autodiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(w1, w2):\n",
    "    return 3 * w1 ** 2 + 2 * w1 * w2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36.000003007075065"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w1, w2 = 5, 3\n",
    "eps = 1e-6\n",
    "(f(w1 + eps, w2) - f(w1, w2)) / eps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10.000000003174137"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(f(w1, w2 + eps) - f(w1, w2)) / eps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "w1, w2 = tf.Variable(5.), tf.Variable(3.)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "gradients = tape.gradient(z, [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GradientTape.gradient can only be called once on non-persistent tapes.\n"
     ]
    }
   ],
   "source": [
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "dz_dw1 = tape.gradient(z, w1)\n",
    "try:\n",
    "    dz_dw2 = tape.gradient(z, w2)\n",
    "except RuntimeError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape(persistent=True) as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "dz_dw1 = tape.gradient(z, w1)\n",
    "dz_dw2 = tape.gradient(z, w2) # works now!\n",
    "del tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>)"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dz_dw1, dz_dw2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "c1, c2 = tf.constant(5.), tf.constant(3.)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(c1, c2)\n",
    "\n",
    "gradients = tape.gradient(z, [c1, c2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None, None]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Forcing the tape to watch any tensors\n",
    "# to record every operation that involves them\n",
    "with tf.GradientTape() as tape:\n",
    "    tape.watch(c1)\n",
    "    tape.watch(c2)\n",
    "    z = f(c1, c2)\n",
    "\n",
    "gradients = tape.gradient(z, [c1, c2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=136.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=30.0>]"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with tf.GradientTape() as tape:\n",
    "    z1 = f(w1, w2 + 2.)\n",
    "    z2 = f(w1, w2 + 5.)\n",
    "    z3 = f(w1, w2 + 7.)\n",
    "\n",
    "tape.gradient([z1, z2, z3], [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape(persistent=True) as tape:\n",
    "    z1 = f(w1, w2 + 2.)\n",
    "    z2 = f(w1, w2 + 5.)\n",
    "    z3 = f(w1, w2 + 7.)\n",
    "\n",
    "tf.reduce_sum(tf.stack([tape.gradient(z, [w1, w2]) for z in (z1, z2, z3)]), axis=0)\n",
    "del tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the individual gradients by calling the tape’s jacobian() method\n",
    "with tf.GradientTape(persistent=True) as hessian_tape:\n",
    "    with tf.GradientTape() as jacobian_tape:\n",
    "        z = f(w1, w2)\n",
    "    jacobians = jacobian_tape.gradient(z, [w1, w2])\n",
    "hessians = [hessian_tape.gradient(jacobian, [w1, w2])\n",
    "            for jacobian in jacobians]\n",
    "del hessian_tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jacobians"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[<tf.Tensor: shape=(), dtype=float32, numpy=6.0>,\n",
       "  <tf.Tensor: shape=(), dtype=float32, numpy=2.0>],\n",
       " [<tf.Tensor: shape=(), dtype=float32, numpy=2.0>, None]]"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hessians"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Variable 'Variable:0' shape=() dtype=float32, numpy=5.0>,\n",
       " <tf.Variable 'Variable:0' shape=() dtype=float32, numpy=3.0>)"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w1, w2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=30.0>, None]"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Stopping gradients from backpropagating through\n",
    "# some part of your neural network\n",
    "def f(w1, w2):\n",
    "    return 3 * w1 ** 2 + tf.stop_gradient(2 * w1 * w2)\n",
    "\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "tape.gradient(z, [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=nan>]"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.Variable(100.)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = my_softplus(x)\n",
    "\n",
    "tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=30.0>"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.math.log(tf.exp(tf.constant(30., dtype=tf.float32)) + 1.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(1,), dtype=float32, numpy=array([nan], dtype=float32)>]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.Variable([100.])\n",
    "with tf.GradientTape() as tape:\n",
    "    z = my_softplus(x)\n",
    "\n",
    "tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.custom_gradient\n",
    "def my_better_softplus(z):\n",
    "    exp = tf.exp(z)\n",
    "    def my_softplus_gradients(grad):\n",
    "        return grad / (1 + 1 / exp)\n",
    "    return tf.math.log(exp + 1), my_softplus_gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_better_softplus(z):\n",
    "    return tf.where(z > 30., z, tf.math.log(tf.exp(z) + 1.))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: shape=(1,), dtype=float32, numpy=array([1000.], dtype=float32)>,\n",
       " [<tf.Tensor: shape=(1,), dtype=float32, numpy=array([nan], dtype=float32)>])"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.Variable([1000.])\n",
    "with tf.GradientTape() as tape:\n",
    "    z = my_better_softplus(x)\n",
    "\n",
    "z, tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Custom Training Loops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "l2_reg = keras.regularizers.l2(0.05)\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"elu\", kernel_initializer=\"he_normal\",\n",
    "                       kernel_regularizer=l2_reg),\n",
    "    keras.layers.Dense(1, kernel_regularizer=l2_reg)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to randomly sample a batch of\n",
    "# instances from the training set\n",
    "def random_batch(X, y, batch_size=32):\n",
    "    idx = np.random.randint(len(X), size=batch_size)\n",
    "    return X[idx], y[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_status_bar(iteration, total, loss, metrics=None):\n",
    "    metrics = \" - \".join([\"{}: {:.4f}\".format(m.name, m.result())\n",
    "                         for m in [loss] + (metrics or [])])\n",
    "    end = \"\" if iteration < total else \"\\n\"\n",
    "    print(\"\\r{}/{} - \".format(iteration, total) + metrics,\n",
    "          end=end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50/50 - loss: 0.0900 - mean_square: 858.5000\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "mean_loss = keras.metrics.Mean(name=\"loss\")\n",
    "mean_square = keras.metrics.Mean(name=\"mean_square\")\n",
    "for i in range(1, 50 + 1):\n",
    "    loss = 1 / i\n",
    "    mean_loss(loss)\n",
    "    mean_square(i ** 2)\n",
    "    print_status_bar(i, 50, mean_loss, [mean_square])\n",
    "    time.sleep(0.05)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A fancier version with a progress bar:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "def progress_bar(iteration, total, size=30):\n",
    "    running = iteration < total\n",
    "    c = \">\" if running else \"=\"\n",
    "    p = (size - 1) * iteration // total\n",
    "    fmt = \"{{:-{}d}}/{{}} [{{}}]\".format(len(str(total)))\n",
    "    params = [iteration, total, \"=\" * p + c + \".\" * (size - p - 1)]\n",
    "    return fmt.format(*params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' 3500/10000 [=>....]'"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "progress_bar(3500, 10000, size=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_status_bar(iteration, total, loss, metrics=None, size=30):\n",
    "    metrics = \" - \".join([\"{}: {:.4f}\".format(m.name, m.result())\n",
    "                         for m in [loss] + (metrics or [])])\n",
    "    end = \"\" if iteration < total else \"\\n\"\n",
    "    print(\"\\r{} - {}\".format(progress_bar(iteration, total), metrics), end=end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50/50 [==============================] - loss: 0.0900 - mean_square: 858.5000\n"
     ]
    }
   ],
   "source": [
    "mean_loss = keras.metrics.Mean(name=\"loss\")\n",
    "mean_square = keras.metrics.Mean(name=\"mean_square\")\n",
    "for i in range(1, 50 + 1):\n",
    "    loss = 1 / i\n",
    "    mean_loss(loss)\n",
    "    mean_square(i ** 2)\n",
    "    print_status_bar(i, 50, mean_loss, [mean_square])\n",
    "    time.sleep(0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining hyperparameters and choosing\n",
    "# the optimizer, the loss function, and the metrics\n",
    "n_epochs = 5\n",
    "batch_size = 32\n",
    "n_steps = len(X_train) // batch_size\n",
    "optimizer = keras.optimizers.Nadam(lr=0.01)\n",
    "loss_fn = keras.losses.mean_squared_error\n",
    "mean_loss = keras.metrics.Mean()\n",
    "metrics = [keras.metrics.MeanAbsoluteError()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "11610/11610 [==============================] - mean: 0.6500 - mean_absolute_error: 0.5223\n",
      "Epoch 2/5\n",
      "11610/11610 [==============================] - mean: 0.6297 - mean_absolute_error: 0.5119\n",
      "Epoch 3/5\n",
      "11610/11610 [==============================] - mean: 0.6258 - mean_absolute_error: 0.5096\n",
      "Epoch 4/5\n",
      "11610/11610 [==============================] - mean: 0.6122 - mean_absolute_error: 0.5052\n",
      "Epoch 5/5\n",
      "11610/11610 [==============================] - mean: 0.6373 - mean_absolute_error: 0.5134\n"
     ]
    }
   ],
   "source": [
    "# Building the custom loop\n",
    "for epoch in range(1, n_epochs + 1):\n",
    "    print(\"Epoch {}/{}\".format(epoch, n_epochs))\n",
    "    for step in range(1, n_steps + 1):\n",
    "        X_batch, y_batch = random_batch(X_train_scaled, y_train)\n",
    "        with tf.GradientTape() as tape:\n",
    "            y_pred = model(X_batch)\n",
    "            main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred))\n",
    "            loss = tf.add_n([main_loss] + model.losses)\n",
    "        gradients = tape.gradient(loss, model.trainable_variables)\n",
    "        optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "        for variable in model.variables:\n",
    "            if variable.constraint is not None:\n",
    "                variable.assign(variable.constraint(variable))\n",
    "        mean_loss(loss)\n",
    "        for metric in metrics:\n",
    "            metric(y_batch, y_pred)\n",
    "        print_status_bar(step * batch_size, len(y_train), mean_loss, metrics)\n",
    "    print_status_bar(len(y_train), len(y_train), mean_loss, metrics)\n",
    "    for metric in [mean_loss] + metrics:\n",
    "        metric.reset_states()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TensorFlow Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert Python function to TensorFlow function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Python function for calculating Square\n",
    "\n",
    "def square(x):\n",
    "    return x ** 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function __main__.square(x)>"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "square"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns square for integer\n",
    "\n",
    "square(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert Python Function to TensorFlow function\n",
    "\n",
    "@tf.function\n",
    "def square(x):\n",
    "    return x ** 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.eager.def_function.Function at 0x7fbc75ce3be0>"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "square"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=4.0>"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns Tensor\n",
    "\n",
    "square(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The Original Python function can be called with python_function attribute\n",
    "\n",
    "square.python_function(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Autograph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def tf__square(x):\n",
      "  do_return = False\n",
      "  retval_ = ag__.UndefinedReturnValue()\n",
      "  with ag__.FunctionScope('square', 'fscope', ag__.ConversionOptions(recursive=True, user_requested=True, optional_features=(), internal_convert_user_code=True)) as fscope:\n",
      "    do_return = True\n",
      "    retval_ = fscope.mark_return_value(x ** 2)\n",
      "  do_return,\n",
      "  return ag__.retval(retval_)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Get the code generated by TensorFlow\n",
    "\n",
    "print(tf.autograph.to_code(square.python_function))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tracing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'pow/y' type=Const>,\n",
       " <tf.Operation 'pow' type=Pow>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# TensorFlow computational graph is wrapped in concrete function get_concrete_function\n",
    "\n",
    "concrete_function = square.get_concrete_function(tf.constant(2.0))\n",
    "ops = concrete_function.graph.get_operations()\n",
    "ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.Variable 'x:0' shape=(3, 3) dtype=float32, numpy=\n",
      "array([[1., 2., 3.],\n",
      "       [4., 5., 6.],\n",
      "       [7., 8., 9.]], dtype=float32)>\n",
      "<tf.Variable 'y:0' shape=(3, 3) dtype=float32, numpy=\n",
      "array([[11., 12., 13.],\n",
      "       [14., 15., 16.],\n",
      "       [17., 18., 19.]], dtype=float32)>\n",
      "tf.Tensor(\n",
      "[[ 1. 12. 13.]\n",
      " [14.  5. 16.]\n",
      " [ 7.  8.  9.]], shape=(3, 3), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# showing tf.where\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "condition = tf.Variable(\n",
    "    np.array([[True, False, False],[False, True, False],[True, True, True]]), dtype = tf.bool, name = 'condition'\n",
    ")\n",
    "\n",
    "x = tf.Variable(\n",
    "    np.array([[1, 2, 3],[4, 5, 6],[7, 8, 9]]), dtype = tf.float32, name = 'x'\n",
    ")\n",
    "y =tf.Variable(\n",
    "    np.array([[11, 12, 13],[14, 15, 16],[17, 18, 19]]), dtype = tf.float32, name = 'y'\n",
    ")\n",
    "\n",
    "print(x)\n",
    "print(y)\n",
    "r = tf.where(condition, x, y)\n",
    "print(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sum using for loop = 4999950000\n",
      "Computation time = 32.964230999994015ms\n",
      "sum using numpy = 4999950000\n",
      "Computation time = 1.2982599999986633ms\n"
     ]
    }
   ],
   "source": [
    "# Vectorized operation vs for loop\n",
    "\n",
    "\n",
    "# With for loop\n",
    "tic = time.process_time()\n",
    "total = 0\n",
    "for i in np.arange(100000):\n",
    "    total = i + total\n",
    "toc = time.process_time()\n",
    "\n",
    "print(\"sum using for loop = \"+ str(total)); \n",
    "print(\"Computation time = \" + str(1000*(toc - tic )) + \"ms\")\n",
    "\n",
    "\n",
    "# With numpy\n",
    "tic = time.process_time()\n",
    "total = np.sum(np.arange(100000))\n",
    "toc = time.process_time()\n",
    "\n",
    "print(\"sum using numpy = \"+ str(total)); \n",
    "print(\"Computation time = \" + str(1000*(toc - tic )) + \"ms\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 4, 6, 8])"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def multiplyby2(x):\n",
    "    y = []\n",
    "    for i in x:\n",
    "        y.append(i)\n",
    "    return x\n",
    "\n",
    "def multiplyby2_np(x):\n",
    "    return x*2\n",
    "\n",
    "\n",
    "multiplyby2_np(np.array([1,2,3,4]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, 3, 4]"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multiplyby2([1,2,3,4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def div2(arr):\n",
    "    # return_1_instead_0\n",
    "    result = arr//2\n",
    "    np.array()\n",
    "    np.where(result == 0, 1, result)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensorflow2",
   "language": "python",
   "name": "tensorflow2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}