Frozen-Lake/Q-Learning/FrozenLake_with_Q_Learning.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5aQKQMJTJBPH"
      },
      "source": [
        "# Q-Learning with FrozenLake 4x4\n",
        "\n",
        "![alt text](http://simoninithomas.com/drlc/Qlearning/frozenlake4x4.png)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "oU8zRXv8QHlm"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import gym\n",
        "import random\n",
        "import time"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0fz-X3HTQueX"
      },
      "source": [
        "## Step 1: Create the environment \n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "mh9jBR_cQ5_a"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\Amin\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\gym\\core.py:329: DeprecationWarning: \u001b[33mWARN: Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  deprecation(\n",
            "c:\\Users\\Amin\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\gym\\wrappers\\step_api_compatibility.py:39: DeprecationWarning: \u001b[33mWARN: Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  deprecation(\n"
          ]
        }
      ],
      "source": [
        "env = gym.make(\"FrozenLake-v1\", render_mode='human')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JEtXMldxQ7uw"
      },
      "source": [
        "## Step 2: Create the Q-table and initialize it \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Uc0xDVd_Q-C8",
        "outputId": "b0bf215c-9ff7-482e-da1e-2fc5dbf46dd0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Action Size:  4 (0: LEFT - 1: DOWN - 2: RIGHT - 3: UP) \n",
            "State Size:  16\n"
          ]
        }
      ],
      "source": [
        "action_size = env.action_space.n\n",
        "state_size = env.observation_space.n\n",
        "print(\"Action Size: \" , action_size, \"(0: LEFT - 1: DOWN - 2: RIGHT - 3: UP)\",\"\\nState Size: \", state_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "17XBPCecAsZG",
        "outputId": "2c3c990c-812c-407b-e82b-647bbd52da24"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Each row is State & Each column is Action\n",
            "------------------\n",
            " [[0. 0. 0. 0.]\n",
            " [0. 0. 0. 0.]\n",
            " [0. 0. 0. 0.]\n",
            " [0. 0. 0. 0.]\n",
            " [0. 0. 0. 0.]\n",
            " [0. 0. 0. 0.]\n",
            " [0. 0. 0. 0.]\n",
            " [0. 0. 0. 0.]\n",
            " [0. 0. 0. 0.]\n",
            " [0. 0. 0. 0.]\n",
            " [0. 0. 0. 0.]\n",
            " [0. 0. 0. 0.]\n",
            " [0. 0. 0. 0.]\n",
            " [0. 0. 0. 0.]\n",
            " [0. 0. 0. 0.]\n",
            " [0. 0. 0. 0.]]\n"
          ]
        }
      ],
      "source": [
        "# Create our Q table with state_size rows and action_size columns (\n",
        "qtable = np.zeros((state_size, action_size))\n",
        "print(\"Each row is State & Each column is Action\\n------------------\\n\", qtable)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WEGeWKKsAu7X"
      },
      "source": [
        "## Step 3: Create the hyperparameters \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "FJhPxx7UAunE"
      },
      "outputs": [],
      "source": [
        "total_episodes = 20000       # Total episodes\n",
        "learning_rate = 0.7          # Learning rate\n",
        "max_steps = 99               # Max steps per episode\n",
        "gamma = 0.95                 # Discounting rate\n",
        "\n",
        "# Exploration parameters\n",
        "epsilon = 1.0                 # Exploration rate\n",
        "max_epsilon = 1.0             # Exploration probability at start\n",
        "min_epsilon = 0.01            # Minimum exploration probability \n",
        "decay_rate = 0.005            # Exponential decay rate for exploration prob"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rcdaN_DbA3ES"
      },
      "source": [
        "## Step 4: The Q-learning algorithm \n",
        "  ![alt text](http://simoninithomas.com/drlc/Qlearning//qtable_algo.png)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5Dqqo_8LA5De",
        "outputId": "ccb047bb-1b5d-45fa-fa54-498cfe05f857"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "---------- Episode: 0 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Step: 12\n",
            "Step: 13\n",
            "Step: 14\n",
            "Step: 15\n",
            "Step: 16\n",
            "Step: 17\n",
            "Step: 18\n",
            "Step: 19\n",
            "\n",
            "---------- Episode: 1 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Step: 12\n",
            "Step: 13\n",
            "Step: 14\n",
            "Step: 15\n",
            "Dead\n",
            "\n",
            "---------- Episode: 2 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Dead\n",
            "\n",
            "---------- Episode: 3 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Dead\n",
            "\n",
            "---------- Episode: 4 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Dead\n",
            "\n",
            "---------- Episode: 5 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Dead\n",
            "\n",
            "---------- Episode: 6 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Dead\n",
            "\n",
            "---------- Episode: 7 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Dead\n",
            "\n",
            "---------- Episode: 8 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Dead\n",
            "\n",
            "---------- Episode: 9 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Dead\n",
            "\n",
            "---------- Episode: 10 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Dead\n",
            "\n",
            "---------- Episode: 11 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Step: 12\n",
            "Step: 13\n",
            "Step: 14\n",
            "WIN\n",
            "\n",
            "---------- Episode: 12 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Dead\n",
            "\n",
            "---------- Episode: 13 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Step: 12\n",
            "Step: 13\n",
            "Step: 14\n",
            "Step: 15\n",
            "Step: 16\n",
            "Step: 17\n",
            "Dead\n",
            "\n",
            "---------- Episode: 14 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Dead\n",
            "\n",
            "---------- Episode: 15 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Dead\n",
            "\n",
            "---------- Episode: 16 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Dead\n",
            "\n",
            "---------- Episode: 17 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Step: 12\n",
            "Step: 13\n",
            "Step: 14\n",
            "Step: 15\n",
            "Step: 16\n",
            "Step: 17\n",
            "Step: 18\n",
            "Step: 19\n",
            "\n",
            "---------- Episode: 18 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Dead\n",
            "\n",
            "---------- Episode: 19 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Dead\n",
            "\n",
            "---------- Episode: 20 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Dead\n",
            "\n",
            "---------- Episode: 21 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Dead\n",
            "\n",
            "---------- Episode: 22 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Dead\n",
            "\n",
            "---------- Episode: 23 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Dead\n",
            "\n",
            "---------- Episode: 24 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Dead\n",
            "\n",
            "---------- Episode: 25 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Dead\n",
            "\n",
            "---------- Episode: 26 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Dead\n",
            "\n",
            "---------- Episode: 27 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Dead\n",
            "\n",
            "---------- Episode: 28 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Dead\n",
            "\n",
            "---------- Episode: 29 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Step: 12\n",
            "Step: 13\n",
            "Dead\n",
            "\n",
            "---------- Episode: 30 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Dead\n",
            "\n",
            "---------- Episode: 31 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Step: 12\n",
            "Step: 13\n",
            "Step: 14\n",
            "Step: 15\n",
            "Step: 16\n",
            "Step: 17\n",
            "Step: 18\n",
            "Step: 19\n",
            "\n",
            "---------- Episode: 32 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Dead\n",
            "\n",
            "---------- Episode: 33 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Dead\n",
            "\n",
            "---------- Episode: 34 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Dead\n",
            "\n",
            "---------- Episode: 35 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Dead\n",
            "\n",
            "---------- Episode: 36 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Dead\n",
            "\n",
            "---------- Episode: 37 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Dead\n",
            "\n",
            "---------- Episode: 38 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Dead\n",
            "\n",
            "---------- Episode: 39 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Dead\n",
            "\n",
            "---------- Episode: 40 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Dead\n",
            "\n",
            "---------- Episode: 41 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Dead\n",
            "\n",
            "---------- Episode: 42 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Step: 12\n",
            "Step: 13\n",
            "Step: 14\n",
            "Step: 15\n",
            "Step: 16\n",
            "Step: 17\n",
            "Step: 18\n",
            "Step: 19\n",
            "\n",
            "---------- Episode: 43 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Step: 12\n",
            "Step: 13\n",
            "Dead\n",
            "\n",
            "---------- Episode: 44 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Dead\n",
            "\n",
            "---------- Episode: 45 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Dead\n",
            "\n",
            "---------- Episode: 46 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Dead\n",
            "\n",
            "---------- Episode: 47 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Dead\n",
            "\n",
            "---------- Episode: 48 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Dead\n",
            "\n",
            "---------- Episode: 49 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Step: 12\n",
            "Step: 13\n",
            "Step: 14\n",
            "Step: 15\n",
            "Step: 16\n",
            "Step: 17\n",
            "Step: 18\n",
            "Dead\n",
            "\n",
            "---------- Episode: 50 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Dead\n",
            "\n",
            "---------- Episode: 51 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Dead\n",
            "\n",
            "---------- Episode: 52 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Dead\n",
            "\n",
            "---------- Episode: 53 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Dead\n",
            "\n",
            "---------- Episode: 54 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Dead\n",
            "\n",
            "---------- Episode: 55 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Dead\n",
            "\n",
            "---------- Episode: 56 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Dead\n",
            "\n",
            "---------- Episode: 57 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Dead\n",
            "\n",
            "---------- Episode: 58 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Dead\n",
            "\n",
            "---------- Episode: 59 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Dead\n",
            "\n",
            "---------- Episode: 60 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Step: 12\n",
            "Step: 13\n",
            "Step: 14\n",
            "Step: 15\n",
            "Step: 16\n",
            "Step: 17\n",
            "Step: 18\n",
            "Step: 19\n",
            "\n",
            "---------- Episode: 61 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Dead\n",
            "\n",
            "---------- Episode: 62 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Dead\n",
            "\n",
            "---------- Episode: 63 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Dead\n",
            "\n",
            "---------- Episode: 64 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Dead\n",
            "\n",
            "---------- Episode: 65 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Dead\n",
            "\n",
            "---------- Episode: 66 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Step: 12\n",
            "Step: 13\n",
            "Step: 14\n",
            "Step: 15\n",
            "Step: 16\n",
            "Step: 17\n",
            "Step: 18\n",
            "Step: 19\n",
            "Dead\n",
            "\n",
            "---------- Episode: 67 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Dead\n",
            "\n",
            "---------- Episode: 68 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Dead\n",
            "\n",
            "---------- Episode: 69 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Step: 12\n",
            "Step: 13\n",
            "WIN\n",
            "\n",
            "---------- Episode: 70 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Step: 12\n",
            "Step: 13\n",
            "Step: 14\n",
            "Dead\n",
            "\n",
            "---------- Episode: 71 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Dead\n",
            "\n",
            "---------- Episode: 72 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Dead\n",
            "\n",
            "---------- Episode: 73 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Dead\n",
            "\n",
            "---------- Episode: 74 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Dead\n",
            "\n",
            "---------- Episode: 75 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Step: 12\n",
            "Step: 13\n",
            "Dead\n",
            "\n",
            "---------- Episode: 76 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Dead\n",
            "\n",
            "---------- Episode: 77 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Dead\n",
            "\n",
            "---------- Episode: 78 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Step: 12\n",
            "Step: 13\n",
            "Step: 14\n",
            "Step: 15\n",
            "Step: 16\n",
            "Dead\n",
            "\n",
            "---------- Episode: 79 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Dead\n",
            "\n",
            "---------- Episode: 80 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Dead\n",
            "\n",
            "---------- Episode: 81 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Dead\n",
            "\n",
            "---------- Episode: 82 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Dead\n",
            "\n",
            "---------- Episode: 83 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Dead\n",
            "\n",
            "---------- Episode: 84 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Dead\n",
            "\n",
            "---------- Episode: 85 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Dead\n",
            "\n",
            "---------- Episode: 86 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Dead\n",
            "\n",
            "---------- Episode: 87 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Step: 12\n",
            "Step: 13\n",
            "Step: 14\n",
            "Step: 15\n",
            "Step: 16\n",
            "Step: 17\n",
            "Step: 18\n",
            "Step: 19\n",
            "\n",
            "---------- Episode: 88 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Dead\n",
            "\n",
            "---------- Episode: 89 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Dead\n",
            "\n",
            "---------- Episode: 90 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Dead\n",
            "\n",
            "---------- Episode: 91 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Dead\n",
            "\n",
            "---------- Episode: 92 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Dead\n",
            "\n",
            "---------- Episode: 93 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Dead\n",
            "\n",
            "---------- Episode: 94 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Step: 12\n",
            "Step: 13\n",
            "Step: 14\n",
            "Step: 15\n",
            "Step: 16\n",
            "WIN\n",
            "\n",
            "---------- Episode: 95 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Dead\n",
            "\n",
            "---------- Episode: 96 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Dead\n",
            "\n",
            "---------- Episode: 97 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Dead\n",
            "\n",
            "---------- Episode: 98 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Step: 8\n",
            "Step: 9\n",
            "Step: 10\n",
            "Step: 11\n",
            "Dead\n",
            "\n",
            "---------- Episode: 99 ----------\n",
            "\n",
            "Step: 0\n",
            "Step: 1\n",
            "Step: 2\n",
            "Step: 3\n",
            "Step: 4\n",
            "Step: 5\n",
            "Step: 6\n",
            "Step: 7\n",
            "Dead\n",
            "Score over time:  0.00015\n",
            "Execution time: 222.024 seconds\n",
            "\n",
            "------------------------------------------\n",
            "\n",
            "[[0.116254266 0.164091572 0.132325925 0.120685028]\n",
            " [0.115975173 0.091025689 0.029127771 0.147197861]\n",
            " [0.129772799 0.123516260 0.142299227 0.137636273]\n",
            " [0.137416588 0.043938467 0.034260854 0.141293096]\n",
            " [0.227130758 0.175571922 0.130245606 0.126834589]\n",
            " [0.000000000 0.000000000 0.000000000 0.000000000]\n",
            " [0.125870505 0.008920420 0.029591406 0.031704682]\n",
            " [0.000000000 0.000000000 0.000000000 0.000000000]\n",
            " [0.097892421 0.082213038 0.039687204 0.236009251]\n",
            " [0.298722987 0.392797704 0.000000000 0.027622910]\n",
            " [0.061230785 0.016326329 0.509032351 0.094625192]\n",
            " [0.000000000 0.000000000 0.000000000 0.000000000]\n",
            " [0.000000000 0.000000000 0.000000000 0.000000000]\n",
            " [0.198650787 0.449207500 0.000000000 0.000000000]\n",
            " [0.309557500 0.895557250 0.547417675 0.612424750]\n",
            " [0.000000000 0.000000000 0.000000000 0.000000000]]\n"
          ]
        }
      ],
      "source": [
        "# 1. List of rewards\n",
        "rewards = []\n",
        "\n",
        "# 2. Until learning is stopped\n",
        "Start = time.time()\n",
        "for episode in range(100):\n",
        "    print(f\"\\n---------- Episode: {episode} ----------\\n\")\n",
        "    # Reset the environment\n",
        "    state = env.reset()\n",
        "    step = 0\n",
        "    done = False\n",
        "    total_rewards = 0\n",
        "    \n",
        "    for step in range(max_steps):\n",
        "        print(f\"Step: {step}\")\n",
        "        # 3. Choose an action a in the current world state (s)\n",
        "        ## First we randomize a number\n",
        "        exp_exp_tradeoff = random.uniform(0, 1)\n",
        "        ## If this number > greater than epsilon --> exploitation (taking the biggest Q value for this state)\n",
        "        if exp_exp_tradeoff > epsilon:\n",
        "            action = np.argmax(qtable[state,:])\n",
        "            #print(exp_exp_tradeoff, \"action\", action)\n",
        "\n",
        "        # Else doing a random choice --> exploration\n",
        "        else:\n",
        "            action = env.action_space.sample()\n",
        "            #print(\"action random\", action)\n",
        "            \n",
        "        \n",
        "        # Take the action (a) and observe the outcome state(s') and reward (r)\n",
        "        new_state, reward, done, info = env.step(action)\n",
        "\n",
        "        # Update Q(s,a):= Q(s,a) + lr [R(s,a) + gamma * max Q(s',a') - Q(s,a)]\n",
        "        # qtable[new_state,:] : all the actions we can take from new state\n",
        "        qtable[state, action] = qtable[state, action] + learning_rate * (reward + gamma * np.max(qtable[new_state, :]) - qtable[state, action])\n",
        "        \n",
        "        total_rewards += reward\n",
        "        \n",
        "        # Our new state is state\n",
        "        state = new_state\n",
        "        \n",
        "        # If done (if we're dead) : finish episode\n",
        "        if done == True and reward == 1: \n",
        "            print(\"WIN\")\n",
        "            break\n",
        "        if done == True: \n",
        "            print(\"Dead\")\n",
        "            break\n",
        "         \n",
        "    # Reduce epsilon (because we need less and less exploration) # https://towardsdatascience.com/learning-rate-schedules-and-adaptive-learning-rate-methods-for-deep-learning-2c8f433990d1\n",
        "    epsilon = min_epsilon + (max_epsilon - min_epsilon)*np.exp(-decay_rate*episode) # https://machinelearningmastery.com/using-learning-rate-schedules-deep-learning-models-python-keras/ \n",
        "    rewards.append(total_rewards)\n",
        "    \n",
        "End = time.time()\n",
        "Time = End - Start\n",
        "print (\"Score over time: \",  sum(rewards)/total_episodes)\n",
        "print('Execution time: {:.3f}'.format(Time), 'seconds')\n",
        "float_formatter = \"{:.9f}\".format # https://stackoverflow.com/questions/21008858/formatting-floats-in-a-numpy-array\n",
        "np.set_printoptions(formatter={'float_kind': float_formatter})\n",
        "print(\"\\n------------------------------------------\\n\")\n",
        "print(qtable)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R5czk9qTBQIU"
      },
      "source": [
        "## Step 5: Use our Q-table to play FrozenLake ! \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Bt8UsREaBNkJ",
        "outputId": "5fafd46d-18dc-491d-87b7-78059ae06567"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "****************************************************\n",
            "EPISODE:  0\n",
            ">> We fell into a hole\n",
            "Number of steps:  23\n",
            "****************************************************\n",
            "EPISODE:  1\n",
            ">> We fell into a hole\n",
            "Number of steps:  17\n",
            "****************************************************\n",
            "EPISODE:  2\n",
            ">> We fell into a hole\n",
            "Number of steps:  33\n",
            "****************************************************\n",
            "EPISODE:  3\n",
            ">> We fell into a hole\n",
            "Number of steps:  9\n",
            "****************************************************\n",
            "EPISODE:  4\n",
            ">> We fell into a hole\n",
            "Number of steps:  18\n",
            "****************************************************\n",
            "EPISODE:  5\n",
            ">> We reached our Goal\n",
            "Number of steps:  8\n",
            "****************************************************\n",
            "EPISODE:  6\n",
            ">> We fell into a hole\n",
            "Number of steps:  9\n",
            "****************************************************\n",
            "EPISODE:  7\n",
            ">> We reached our Goal\n",
            "Number of steps:  16\n",
            "****************************************************\n",
            "EPISODE:  8\n",
            ">> We fell into a hole\n",
            "Number of steps:  11\n",
            "****************************************************\n",
            "EPISODE:  9\n",
            ">> We fell into a hole\n",
            "Number of steps:  9\n",
            "****************************************************\n",
            "EPISODE:  10\n",
            ">> We fell into a hole\n",
            "Number of steps:  15\n",
            "****************************************************\n",
            "EPISODE:  11\n",
            ">> We fell into a hole\n",
            "Number of steps:  24\n",
            "****************************************************\n",
            "EPISODE:  12\n",
            ">> We reached our Goal\n",
            "Number of steps:  14\n",
            "****************************************************\n",
            "EPISODE:  13\n",
            ">> We fell into a hole\n",
            "Number of steps:  22\n",
            "****************************************************\n",
            "EPISODE:  14\n",
            ">> We fell into a hole\n",
            "Number of steps:  9\n",
            "****************************************************\n",
            "EPISODE:  15\n",
            ">> We reached our Goal\n",
            "Number of steps:  8\n",
            "****************************************************\n",
            "EPISODE:  16\n",
            ">> We fell into a hole\n",
            "Number of steps:  12\n",
            "****************************************************\n",
            "EPISODE:  17\n",
            ">> We fell into a hole\n",
            "Number of steps:  17\n",
            "****************************************************\n",
            "EPISODE:  18\n",
            ">> We fell into a hole\n",
            "Number of steps:  15\n",
            "****************************************************\n",
            "EPISODE:  19\n",
            ">> We fell into a hole\n",
            "Number of steps:  30\n",
            "****************************************************\n",
            "EPISODE:  20\n",
            ">> We reached our Goal\n",
            "Number of steps:  11\n",
            "****************************************************\n",
            "EPISODE:  21\n",
            ">> We fell into a hole\n",
            "Number of steps:  34\n",
            "****************************************************\n",
            "EPISODE:  22\n",
            ">> We reached our Goal\n",
            "Number of steps:  45\n",
            "****************************************************\n",
            "EPISODE:  23\n",
            ">> We fell into a hole\n",
            "Number of steps:  43\n",
            "****************************************************\n",
            "EPISODE:  24\n",
            ">> We reached our Goal\n",
            "Number of steps:  25\n",
            "****************************************************\n",
            "EPISODE:  25\n",
            ">> We fell into a hole\n",
            "Number of steps:  25\n",
            "****************************************************\n",
            "EPISODE:  26\n",
            ">> We fell into a hole\n",
            "Number of steps:  8\n",
            "****************************************************\n",
            "EPISODE:  27\n",
            ">> We fell into a hole\n",
            "Number of steps:  6\n",
            "****************************************************\n",
            "EPISODE:  28\n",
            ">> We fell into a hole\n",
            "Number of steps:  15\n",
            "****************************************************\n",
            "EPISODE:  29\n",
            ">> We fell into a hole\n",
            "Number of steps:  4\n",
            "****************************************************\n",
            "EPISODE:  30\n",
            ">> We fell into a hole\n",
            "Number of steps:  5\n",
            "****************************************************\n",
            "EPISODE:  31\n",
            ">> We fell into a hole\n",
            "Number of steps:  42\n",
            "****************************************************\n",
            "EPISODE:  32\n",
            ">> We fell into a hole\n",
            "Number of steps:  11\n",
            "****************************************************\n",
            "EPISODE:  33\n",
            ">> We fell into a hole\n",
            "Number of steps:  25\n",
            "****************************************************\n",
            "EPISODE:  34\n",
            ">> We fell into a hole\n",
            "Number of steps:  25\n",
            "****************************************************\n",
            "EPISODE:  35\n",
            ">> We fell into a hole\n",
            "Number of steps:  7\n",
            "****************************************************\n",
            "EPISODE:  36\n",
            ">> We fell into a hole\n",
            "Number of steps:  16\n",
            "****************************************************\n",
            "EPISODE:  37\n",
            ">> We fell into a hole\n",
            "Number of steps:  11\n",
            "****************************************************\n",
            "EPISODE:  38\n",
            ">> We fell into a hole\n",
            "Number of steps:  18\n",
            "****************************************************\n",
            "EPISODE:  39\n",
            ">> We fell into a hole\n",
            "Number of steps:  20\n",
            "****************************************************\n",
            "EPISODE:  40\n",
            ">> We fell into a hole\n",
            "Number of steps:  47\n",
            "****************************************************\n",
            "EPISODE:  41\n",
            ">> We reached our Goal\n",
            "Number of steps:  31\n",
            "****************************************************\n",
            "EPISODE:  42\n",
            ">> We reached our Goal\n",
            "Number of steps:  9\n",
            "****************************************************\n",
            "EPISODE:  43\n",
            ">> We fell into a hole\n",
            "Number of steps:  16\n",
            "****************************************************\n",
            "EPISODE:  44\n",
            ">> We fell into a hole\n",
            "Number of steps:  27\n",
            "****************************************************\n",
            "EPISODE:  45\n",
            ">> We reached our Goal\n",
            "Number of steps:  7\n",
            "****************************************************\n",
            "EPISODE:  46\n",
            ">> We fell into a hole\n",
            "Number of steps:  8\n",
            "****************************************************\n",
            "EPISODE:  47\n",
            ">> We fell into a hole\n",
            "Number of steps:  11\n",
            "****************************************************\n",
            "EPISODE:  48\n",
            ">> We fell into a hole\n",
            "Number of steps:  19\n",
            "****************************************************\n",
            "EPISODE:  49\n",
            ">> We fell into a hole\n",
            "Number of steps:  11\n",
            "****************************************************\n",
            "EPISODE:  50\n",
            ">> We fell into a hole\n",
            "Number of steps:  10\n",
            "****************************************************\n",
            "EPISODE:  51\n",
            ">> We fell into a hole\n",
            "Number of steps:  14\n",
            "****************************************************\n",
            "EPISODE:  52\n",
            ">> We reached our Goal\n",
            "Number of steps:  11\n",
            "****************************************************\n",
            "EPISODE:  53\n",
            ">> We fell into a hole\n",
            "Number of steps:  11\n",
            "****************************************************\n",
            "EPISODE:  54\n",
            ">> We fell into a hole\n",
            "Number of steps:  8\n",
            "****************************************************\n",
            "EPISODE:  55\n",
            ">> We reached our Goal\n",
            "Number of steps:  15\n",
            "****************************************************\n",
            "EPISODE:  56\n",
            ">> We fell into a hole\n",
            "Number of steps:  8\n",
            "****************************************************\n",
            "EPISODE:  57\n",
            ">> We reached our Goal\n",
            "Number of steps:  22\n",
            "****************************************************\n",
            "EPISODE:  58\n",
            ">> We fell into a hole\n",
            "Number of steps:  5\n",
            "****************************************************\n",
            "EPISODE:  59\n",
            ">> We reached our Goal\n",
            "Number of steps:  13\n",
            "****************************************************\n",
            "EPISODE:  60\n",
            ">> We fell into a hole\n",
            "Number of steps:  32\n",
            "****************************************************\n",
            "EPISODE:  61\n",
            ">> We fell into a hole\n",
            "Number of steps:  9\n",
            "****************************************************\n",
            "EPISODE:  62\n",
            ">> We fell into a hole\n",
            "Number of steps:  8\n",
            "****************************************************\n",
            "EPISODE:  63\n",
            ">> We reached our Goal\n",
            "Number of steps:  14\n",
            "****************************************************\n",
            "EPISODE:  64\n",
            ">> We fell into a hole\n",
            "Number of steps:  28\n",
            "****************************************************\n",
            "EPISODE:  65\n",
            ">> We fell into a hole\n",
            "Number of steps:  10\n",
            "****************************************************\n",
            "EPISODE:  66\n",
            ">> We fell into a hole\n",
            "Number of steps:  10\n",
            "****************************************************\n",
            "EPISODE:  67\n",
            ">> We fell into a hole\n",
            "Number of steps:  11\n",
            "****************************************************\n",
            "EPISODE:  68\n",
            ">> We fell into a hole\n",
            "Number of steps:  11\n",
            "****************************************************\n",
            "EPISODE:  69\n",
            ">> We fell into a hole\n",
            "Number of steps:  13\n",
            "****************************************************\n",
            "EPISODE:  70\n",
            ">> We fell into a hole\n",
            "Number of steps:  6\n",
            "****************************************************\n",
            "EPISODE:  71\n",
            ">> We fell into a hole\n",
            "Number of steps:  29\n",
            "****************************************************\n",
            "EPISODE:  72\n",
            ">> We fell into a hole\n",
            "Number of steps:  12\n",
            "****************************************************\n",
            "EPISODE:  73\n",
            ">> We fell into a hole\n",
            "Number of steps:  15\n",
            "****************************************************\n",
            "EPISODE:  74\n",
            ">> We fell into a hole\n",
            "Number of steps:  6\n",
            "****************************************************\n",
            "EPISODE:  75\n",
            ">> We reached our Goal\n",
            "Number of steps:  27\n",
            "****************************************************\n",
            "EPISODE:  76\n",
            ">> We reached our Goal\n",
            "Number of steps:  29\n",
            "****************************************************\n",
            "EPISODE:  77\n",
            ">> We fell into a hole\n",
            "Number of steps:  31\n",
            "****************************************************\n",
            "EPISODE:  78\n",
            ">> We fell into a hole\n",
            "Number of steps:  7\n",
            "****************************************************\n",
            "EPISODE:  79\n",
            ">> We fell into a hole\n",
            "Number of steps:  3\n",
            "****************************************************\n",
            "EPISODE:  80\n",
            ">> We fell into a hole\n",
            "Number of steps:  35\n",
            "****************************************************\n",
            "EPISODE:  81\n",
            ">> We fell into a hole\n",
            "Number of steps:  18\n",
            "****************************************************\n",
            "EPISODE:  82\n",
            ">> We fell into a hole\n",
            "Number of steps:  14\n",
            "****************************************************\n",
            "EPISODE:  83\n",
            ">> We fell into a hole\n",
            "Number of steps:  18\n",
            "****************************************************\n",
            "EPISODE:  84\n",
            ">> We fell into a hole\n",
            "Number of steps:  10\n",
            "****************************************************\n",
            "EPISODE:  85\n",
            ">> We fell into a hole\n",
            "Number of steps:  9\n",
            "****************************************************\n",
            "EPISODE:  86\n",
            ">> We fell into a hole\n",
            "Number of steps:  9\n",
            "****************************************************\n",
            "EPISODE:  87\n",
            ">> We fell into a hole\n",
            "Number of steps:  36\n",
            "****************************************************\n",
            "EPISODE:  88\n",
            ">> We fell into a hole\n",
            "Number of steps:  3\n",
            "****************************************************\n",
            "EPISODE:  89\n",
            ">> We fell into a hole\n",
            "Number of steps:  18\n",
            "****************************************************\n",
            "EPISODE:  90\n",
            ">> We fell into a hole\n",
            "Number of steps:  46\n",
            "****************************************************\n",
            "EPISODE:  91\n",
            ">> We fell into a hole\n",
            "Number of steps:  11\n",
            "****************************************************\n",
            "EPISODE:  92\n",
            ">> We fell into a hole\n",
            "Number of steps:  30\n",
            "****************************************************\n",
            "EPISODE:  93\n",
            ">> We fell into a hole\n",
            "Number of steps:  12\n",
            "****************************************************\n",
            "EPISODE:  94\n",
            ">> We fell into a hole\n",
            "Number of steps:  5\n",
            "****************************************************\n",
            "EPISODE:  95\n",
            ">> We fell into a hole\n",
            "Number of steps:  20\n",
            "****************************************************\n",
            "EPISODE:  96\n",
            ">> We fell into a hole\n",
            "Number of steps:  16\n",
            "****************************************************\n",
            "EPISODE:  97\n",
            ">> We fell into a hole\n",
            "Number of steps:  21\n",
            "****************************************************\n",
            "EPISODE:  98\n",
            ">> We fell into a hole\n",
            "Number of steps:  33\n",
            "****************************************************\n",
            "EPISODE:  99\n",
            ">> We fell into a hole\n",
            "Number of steps:  15\n",
            "****************************************************\n",
            "\n",
            "Total Episode: 100\n",
            "Max Step per Episode: 99\n",
            "Win: 17\n",
            "Loss: 83\n"
          ]
        }
      ],
      "source": [
        "env.reset()\n",
        "max_steps = 99\n",
        "n = 100\n",
        "win = 0\n",
        "loss = 0\n",
        "for episode in range(n):\n",
        "    state = env.reset()\n",
        "    step = 0\n",
        "    done = False\n",
        "    print(\"****************************************************\")\n",
        "    print(\"EPISODE: \", episode)\n",
        "\n",
        "    for step in range(max_steps):\n",
        "        # Take the action (index) that have the maximum expected future reward given that state\n",
        "        action = np.argmax(qtable[state,:])\n",
        "        \n",
        "        new_state, reward, done, info = env.step(action)\n",
        "        \n",
        "        if done:\n",
        "            # Here, we decide to only print the last state (to see if our agent is on the goal or fall into an hole)\n",
        "            env.render()\n",
        "            if new_state == 15:\n",
        "                win += 1\n",
        "                print(\">> We reached our Goal\")\n",
        "            else:\n",
        "                loss += 1\n",
        "                print(\">> We fell into a hole\")\n",
        "            \n",
        "            # We print the number of step it took.\n",
        "            print(\"Number of steps: \", step)\n",
        "            \n",
        "            break\n",
        "        state = new_state\n",
        "env.close()\n",
        "print(\"****************************************************\")\n",
        "print(f\"\\nTotal Episode: {n}\\nMax Step per Episode: {max_steps}\\nWin: {win}\\nLoss: {loss}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "name": "FrozenLake-with-Q-Learning.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3.10.4 64-bit",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.4"
    },
    "vscode": {
      "interpreter": {
        "hash": "aafc33a19d1e6c143b5a4b283b91d6d8d620b2aadbd9675105175468fd18e65b"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}