MIT_TF_From_the_Ground_Up_Exercises_2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "MIT: TF From the Ground Up Exercises 2.ipynb",
      "private_outputs": true,
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "au5kdSKU9ddV"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's make some variables"
      ],
      "metadata": {
        "id": "mALIj8cr9p1K"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "my_tensor = tf.constant([[1.0, 2.0], [3.0, 4.0]])\n",
        "my_variable = tf.Variable(my_tensor)\n",
        "\n",
        "# Variables can be all kinds of types, just like tensors\n",
        "bool_variable = tf.Variable([False, False, False, True])\n",
        "complex_variable = tf.Variable([5 + 4j, 6 + 1j])\n"
      ],
      "metadata": {
        "id": "e6mA7fmr9luu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Change variables"
      ],
      "metadata": {
        "id": "FnpPlqx7937X"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "a = tf.Variable([2.0, 3.0])\n",
        "# This will keep the same dtype, float32\n",
        "a.assign([1, 2])\n",
        "# Not allowed as it resizes the variable:\n",
        "try:\n",
        " a.assign([1.0, 2.0, 3.0])\n",
        "except Exception as e:\n",
        " print(f\"{type(e).__name__}: {e}\")\n"
      ],
      "metadata": {
        "id": "ILZjkO0or89j"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Delete a variable using Python lifespans."
      ],
      "metadata": {
        "id": "Y0DwyJrtIGEA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "w = tf.Variable([1.0, 2.0, 3.0])\n",
        "b = tf.Variable(2.0)\n",
        "x = tf.constant([2.0, 3.0, 4.0])\n",
        "\n",
        "y = w * x + b # <tf.Tensor: [ 4.,  8., 14.], dtype=float32)>\n",
        "\n",
        "# Explicitly remove the Python object\n",
        "del(w)\n"
      ],
      "metadata": {
        "id": "U77xW-VTIJGH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Calculus"
      ],
      "metadata": {
        "id": "xzQrtTCj90ug"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "x = tf.Variable([3.0, 3.0])\n",
        "\n",
        "# y = x ** 2\n",
        "y = x**2 # output [9.0, 9.0]\n",
        "\n",
        "print(y)"
      ],
      "metadata": {
        "id": "o9clpG3b910J"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x = tf.Variable([3.0, 3.0])\n",
        "\n",
        "with tf.GradientTape() as tape:\n",
        "  y = x**2 # output [9.0, 9.0]\n",
        "\n",
        "# dy = 2x * dx\n",
        "dy_dx = tape.gradient(y, x)\n",
        "print(dy_dx.numpy())  # output [6.0, 6.0]"
      ],
      "metadata": {
        "id": "OfUO5RiyIX1X"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Gradient tapes for a small model."
      ],
      "metadata": {
        "id": "Tt8i_lxOr_2J"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "layer = tf.keras.layers.Dense(2, activation='relu')\n",
        "x = tf.constant([[1., 2., 3.]])\n",
        "\n",
        "with tf.GradientTape() as tape:\n",
        " # Forward pass\n",
        " y = layer(x)\n",
        " loss = tf.reduce_mean(y**2)\n",
        "\n",
        "# Calculate gradients with respect to every trainable variable\n",
        "grad = tape.gradient(loss, layer.trainable_variables)\n",
        "for var, g in zip(layer.trainable_variables, grad):\n",
        " print(f'{var.name}, shape: {g.shape}')"
      ],
      "metadata": {
        "id": "DZk09oonCpM8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Gradients from intermediate values."
      ],
      "metadata": {
        "id": "pkQ-qMyRCtsj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "x = tf.constant(3.0)\n",
        "\n",
        "with tf.GradientTape() as tape:\n",
        " tape.watch(x)\n",
        " r = x * x\n",
        " z = r * r\n",
        "\n",
        "# Use the tape to compute the gradient of z with respect to the\n",
        "# intermediate value r.\n",
        "# dz_dx = 2 * r, where r = x ** 2\n",
        "print(tape.gradient(z, r).numpy()) # 18"
      ],
      "metadata": {
        "id": "k3vTzutzIlYb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x = tf.constant(1.0)\n",
        "v0 = tf.Variable(2.0)\n",
        "v1 = tf.Variable(2.0)\n",
        "\n",
        "with tf.GradientTape(persistent=True) as tape:\n",
        " tape.watch(x)\n",
        " if x > 0.0:\n",
        "   result = v0\n",
        " else:\n",
        "   result = v1**2\n",
        "\n",
        "dv0, dv1 = tape.gradient(result, [v0, v1]) # 1.0, None\n",
        "print(dv0, dv1)"
      ],
      "metadata": {
        "id": "R26j9wwaIo8N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Lots more details, like:\n",
        "* Watching\n",
        "* Stopping gradient computation\n",
        "* Gradients of non-scalar targets\n",
        "* Jacobians\n",
        "* 2nd order gradients\n"
      ],
      "metadata": {
        "id": "XyJrysiGItoc"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Exercise: y = mx + b\n",
        "\n",
        "Try writing a function, y = mx + b.\n",
        "\n",
        "Send some float tensors into it.\n",
        "\n"
      ],
      "metadata": {
        "id": "kfA3kyDJI5mx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def yeqmxb(x, m, b):\n",
        "  return (m * x) + b\n",
        "\n",
        "\n",
        "the_x = tf.constant(2.0)\n",
        "the_m = tf.constant(2.0)\n",
        "the_b = tf.constant(4.0)\n",
        "\n",
       "def yeqmxb(x, m, b):\n",
        "  return m * x + b\n",
        "\n",
        "y = yeqmxb(the_x, the_m, the_b)\n",
        "\n",
        "print(y)",
        "\n"
      ],
      "metadata": {
        "id": "VkGDtQ75ItBq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Can you figure out the gradient?"
      ],
      "metadata": {
        "id": "hhrLV3WsoXOP"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Solution"
      ],
      "metadata": {
        "id": "CgikKY45oTr5"
      }
    },
    {
      "cell_type": "code",
      "source": [
         "with tf.GradientTape() as tape:\n",
        "  tape.watch(the_x)\n",
        "  y = yeqmxb(the_x, the_m, the_b)\n",
        "\n",
        "tape.gradient(y, [the_x])"
      ],
      "metadata": {
        "id": "r3nOCylNlzPf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now, find the gradient of y with respect to x!"
      ],
      "metadata": {
        "id": "SdygRcUAC1lH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with tf.GradientTape() as tape:\n",
        "  tape.watch(the_x)\n",
        "  y = yeqmxb(the_x, the_b, the_b)\n",
        "\n",
        "tape.gradient(y, the_x)"
      ],
      "metadata": {
        "id": "RpbJKnVH0CAG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Bonus: Why does this return `None` as a derivative?"
      ],
      "metadata": {
        "id": "wUiySrlq6jhX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "another_x = tf.constant(40.)\n",
        "another_m = tf.constant(30.)\n",
        "another_b = tf.constant(20.)\n",
        "\n",
        "with tf.GradientTape() as tape:\n",
        "  tape.watch(another_x)\n",
        "  y = yeqmxb(another_x, another_m, another_b)\n",
        "\n",
        "print(tape.gradient(y, [another_x]))"
      ],
      "metadata": {
        "id": "gDpmGdro0EHs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Render batch"
      ],
      "metadata": {
        "id": "35r3jeKd0F2y"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}