Fuzzy_vs_PPO.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMnEv8Va0xLJuCdKTAxtRP6",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Perfect-Cube/Volkswagon-imobilothon-4.0/blob/main/Fuzzy_vs_PPO.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install scikit-fuzzy\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RspcWr9pCJ-P",
        "outputId": "1c09af22-4942-4349-d3b7-bf4ea9d860db"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting scikit-fuzzy\n",
            "  Downloading scikit_fuzzy-0.5.0-py2.py3-none-any.whl.metadata (2.6 kB)\n",
            "Downloading scikit_fuzzy-0.5.0-py2.py3-none-any.whl (920 kB)\n",
            "\u001b[?25l   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/920.8 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K   \u001b[91m━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m153.6/920.8 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K   \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m \u001b[32m911.4/920.8 kB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m920.8/920.8 kB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: scikit-fuzzy\n",
            "Successfully installed scikit-fuzzy-0.5.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oPjF4ZccCFH3",
        "outputId": "3d94992e-368c-4a4e-fa96-3bcda273bf0c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Temperature: 60, Charge: 80, Cooling Power: 49.99999999999997\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "import skfuzzy as fuzz\n",
        "from skfuzzy import control as ctrl\n",
        "\n",
        "# Define fuzzy variables for temperature, charge, and cooling power\n",
        "temperature = ctrl.Antecedent(np.arange(0, 101, 1), 'temperature')\n",
        "charge = ctrl.Antecedent(np.arange(0, 101, 1), 'charge')\n",
        "cooling_power = ctrl.Consequent(np.arange(0, 101, 1), 'cooling_power')\n",
        "\n",
        "# Define fuzzy sets and membership functions\n",
        "temperature['low'] = fuzz.trimf(temperature.universe, [0, 0, 40])\n",
        "temperature['medium'] = fuzz.trimf(temperature.universe, [30, 50, 70])\n",
        "temperature['high'] = fuzz.trimf(temperature.universe, [60, 100, 100])\n",
        "\n",
        "charge['low'] = fuzz.trimf(charge.universe, [0, 0, 50])\n",
        "charge['medium'] = fuzz.trimf(charge.universe, [30, 50, 80])\n",
        "charge['high'] = fuzz.trimf(charge.universe, [70, 100, 100])\n",
        "\n",
        "cooling_power['low'] = fuzz.trimf(cooling_power.universe, [0, 0, 50])\n",
        "cooling_power['medium'] = fuzz.trimf(cooling_power.universe, [25, 50, 75])\n",
        "cooling_power['high'] = fuzz.trimf(cooling_power.universe, [50, 100, 100])\n",
        "\n",
        "# Define fuzzy rules\n",
        "rule1 = ctrl.Rule(temperature['high'] & charge['high'], cooling_power['high'])\n",
        "rule2 = ctrl.Rule(temperature['medium'] & charge['high'], cooling_power['medium'])\n",
        "rule3 = ctrl.Rule(temperature['low'] | charge['low'], cooling_power['low'])\n",
        "\n",
        "# Control system\n",
        "cooling_ctrl = ctrl.ControlSystem([rule1, rule2, rule3])\n",
        "cooling_sim = ctrl.ControlSystemSimulation(cooling_ctrl)\n",
        "\n",
        "# Test the fuzzy controller\n",
        "temp_input = 60  # Example temperature\n",
        "charge_input = 80  # Example charge\n",
        "\n",
        "cooling_sim.input['temperature'] = temp_input\n",
        "cooling_sim.input['charge'] = charge_input\n",
        "\n",
        "# Compute output\n",
        "cooling_sim.compute()\n",
        "print(f\"Temperature: {temp_input}, Charge: {charge_input}, Cooling Power: {cooling_sim.output['cooling_power']}\")\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install stable-baselines3 gym\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qvbrl3v5CWeT",
        "outputId": "c24654c4-70a4-4383-9626-a15fb5f83e86"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting stable-baselines3\n",
            "  Downloading stable_baselines3-2.3.2-py3-none-any.whl.metadata (5.1 kB)\n",
            "Requirement already satisfied: gym in /usr/local/lib/python3.10/dist-packages (0.25.2)\n",
            "Collecting gymnasium<0.30,>=0.28.1 (from stable-baselines3)\n",
            "  Downloading gymnasium-0.29.1-py3-none-any.whl.metadata (10 kB)\n",
            "Requirement already satisfied: numpy>=1.20 in /usr/local/lib/python3.10/dist-packages (from stable-baselines3) (1.26.4)\n",
            "Requirement already satisfied: torch>=1.13 in /usr/local/lib/python3.10/dist-packages (from stable-baselines3) (2.5.0+cu121)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.10/dist-packages (from stable-baselines3) (3.1.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from stable-baselines3) (2.2.2)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from stable-baselines3) (3.8.0)\n",
            "Requirement already satisfied: gym-notices>=0.0.4 in /usr/local/lib/python3.10/dist-packages (from gym) (0.0.8)\n",
            "Requirement already satisfied: typing-extensions>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from gymnasium<0.30,>=0.28.1->stable-baselines3) (4.12.2)\n",
            "Collecting farama-notifications>=0.0.1 (from gymnasium<0.30,>=0.28.1->stable-baselines3)\n",
            "  Downloading Farama_Notifications-0.0.4-py3-none-any.whl.metadata (558 bytes)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.13->stable-baselines3) (3.16.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.13->stable-baselines3) (3.4.2)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13->stable-baselines3) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.13->stable-baselines3) (2024.10.0)\n",
            "Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13->stable-baselines3) (1.13.1)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy==1.13.1->torch>=1.13->stable-baselines3) (1.3.0)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->stable-baselines3) (1.3.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->stable-baselines3) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->stable-baselines3) (4.54.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->stable-baselines3) (1.4.7)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->stable-baselines3) (24.1)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->stable-baselines3) (10.4.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->stable-baselines3) (3.2.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib->stable-baselines3) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->stable-baselines3) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->stable-baselines3) (2024.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib->stable-baselines3) (1.16.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.13->stable-baselines3) (3.0.2)\n",
            "Downloading stable_baselines3-2.3.2-py3-none-any.whl (182 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m182.3/182.3 kB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading gymnasium-0.29.1-py3-none-any.whl (953 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m953.9/953.9 kB\u001b[0m \u001b[31m24.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading Farama_Notifications-0.0.4-py3-none-any.whl (2.5 kB)\n",
            "Installing collected packages: farama-notifications, gymnasium, stable-baselines3\n",
            "Successfully installed farama-notifications-0.0.4 gymnasium-0.29.1 stable-baselines3-2.3.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import gym\n",
        "from stable_baselines3 import PPO\n",
        "\n",
        "# Define a custom environment (simple example for demonstration purposes)\n",
        "class BatteryEnv(gym.Env):\n",
        "    def __init__(self):\n",
        "        super(BatteryEnv, self).__init__()\n",
        "        # Define action and observation space\n",
        "        # Action: Cooling power (continuous between 0 and 1)\n",
        "        self.action_space = gym.spaces.Box(low=0, high=1, shape=(1,), dtype=float)\n",
        "        # Observation: Battery temperature and state of charge\n",
        "        self.observation_space = gym.spaces.Box(low=0, high=100, shape=(2,), dtype=float)\n",
        "        self.state = [50, 80]  # Example: [temperature, charge]\n",
        "\n",
        "    def reset(self):\n",
        "        self.state = [50, 80]  # Reset to initial conditions\n",
        "        return self.state\n",
        "\n",
        "    def step(self, action):\n",
        "        temp, charge = self.state\n",
        "        cooling_power = action[0]  # Action is cooling power (0 to 1)\n",
        "\n",
        "        # Update battery temperature and charge (simple dynamics for demonstration)\n",
        "        temp -= cooling_power * 5  # Cooling reduces temperature\n",
        "        charge -= 0.1 * (1 - cooling_power)  # Charge decreases with less cooling\n",
        "\n",
        "        # Update state\n",
        "        self.state = [temp, charge]\n",
        "\n",
        "        # Calculate reward (e.g., keeping temp between 20 and 60 and maintaining charge)\n",
        "        reward = -abs(temp - 40) - abs(charge - 80)\n",
        "\n",
        "        # Done if battery reaches critical levels\n",
        "        done = temp < 20 or temp > 80 or charge < 10\n",
        "\n",
        "        return self.state, reward, done, {}\n",
        "\n",
        "env = BatteryEnv()\n",
        "\n",
        "# Initialize PPO model\n",
        "model = PPO(\"MlpPolicy\", env, verbose=1)\n",
        "# Train the model\n",
        "model.learn(total_timesteps=10000)\n",
        "\n",
        "# Test the model\n",
        "obs = env.reset()\n",
        "for _ in range(100):\n",
        "    action, _states = model.predict(obs, deterministic=True)\n",
        "    obs, reward, done, info = env.step(action)\n",
        "    print(f\"Obs: {obs}, Reward: {reward}, Done: {done}\")\n",
        "    if done:\n",
        "        break\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s2Cg3BPUCZJi",
        "outputId": "0298d69d-efee-4514-ea5d-f07fbe0a435b"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using cpu device\n",
            "Wrapping the env with a `Monitor` wrapper\n",
            "Wrapping the env in a DummyVecEnv.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/stable_baselines3/common/vec_env/patch_gym.py:49: UserWarning: You provided an OpenAI Gym environment. We strongly recommend transitioning to Gymnasium environments. Stable-Baselines3 is automatically wrapping your environments in a compatibility layer, which could potentially cause issues.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 19.8     |\n",
            "|    ep_rew_mean     | -193     |\n",
            "| time/              |          |\n",
            "|    fps             | 1292     |\n",
            "|    iterations      | 1        |\n",
            "|    time_elapsed    | 1        |\n",
            "|    total_timesteps | 2048     |\n",
            "---------------------------------\n",
            "------------------------------------------\n",
            "| rollout/                |              |\n",
            "|    ep_len_mean          | 18.4         |\n",
            "|    ep_rew_mean          | -175         |\n",
            "| time/                   |              |\n",
            "|    fps                  | 818          |\n",
            "|    iterations           | 2            |\n",
            "|    time_elapsed         | 5            |\n",
            "|    total_timesteps      | 4096         |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0080325045 |\n",
            "|    clip_fraction        | 0.109        |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -1.41        |\n",
            "|    explained_variance   | -0.00199     |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 3.56e+03     |\n",
            "|    n_updates            | 10           |\n",
            "|    policy_gradient_loss | -0.0164      |\n",
            "|    std                  | 0.985        |\n",
            "|    value_loss           | 7.4e+03      |\n",
            "------------------------------------------\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 14.9        |\n",
            "|    ep_rew_mean          | -145        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 739         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 8           |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.012717681 |\n",
            "|    clip_fraction        | 0.166       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.4        |\n",
            "|    explained_variance   | 0.000402    |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 2.85e+03    |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.02       |\n",
            "|    std                  | 0.972       |\n",
            "|    value_loss           | 5.8e+03     |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 14          |\n",
            "|    ep_rew_mean          | -130        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 722         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 11          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.010866648 |\n",
            "|    clip_fraction        | 0.133       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.37       |\n",
            "|    explained_variance   | 0.000118    |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 1.86e+03    |\n",
            "|    n_updates            | 30          |\n",
            "|    policy_gradient_loss | -0.02       |\n",
            "|    std                  | 0.934       |\n",
            "|    value_loss           | 4.06e+03    |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 13          |\n",
            "|    ep_rew_mean          | -116        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 717         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 14          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016884059 |\n",
            "|    clip_fraction        | 0.184       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.32       |\n",
            "|    explained_variance   | 4.64e-05    |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 1.61e+03    |\n",
            "|    n_updates            | 40          |\n",
            "|    policy_gradient_loss | -0.0222     |\n",
            "|    std                  | 0.881       |\n",
            "|    value_loss           | 3.2e+03     |\n",
            "-----------------------------------------\n",
            "Obs: [45.94295918941498, 79.9811408162117], Reward: -5.961818373203272, Done: False\n",
            "Obs: [42.60864019393921, 79.94782719612122], Reward: -2.6608129978179846, Done: False\n",
            "Obs: [40.49386456608772, 79.89012270867825], Reward: -0.6037418574094744, Done: False\n",
            "Obs: [38.88475805521011, 79.8223048388958], Reward: -1.292937105894083, Done: False\n",
            "Obs: [37.388697415590286, 79.7522260516882], Reward: -2.8590765327215166, Done: False\n",
            "Obs: [35.76646327972412, 79.68467073440551], Reward: -4.548865985870364, Done: False\n",
            "Obs: [33.66197794675827, 79.62676044106483], Reward: -6.711261612176898, Done: False\n",
            "Obs: [30.452010929584503, 79.5909597814083], Reward: -9.957029289007195, Done: False\n",
            "Obs: [26.139174103736877, 79.57721651792525], Reward: -14.283609378337871, Done: False\n",
            "Obs: [21.553346514701843, 79.56893306970595], Reward: -18.877720415592208, Done: False\n",
            "Obs: [17.102108597755432, 79.55795782804488], Reward: -23.33993357419969, Done: True\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install shimmy"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 497
        },
        "id": "gYTY63UfCgc9",
        "outputId": "e159d18c-901e-4e42-822e-99e2079e1fd5"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting shimmy\n",
            "  Downloading Shimmy-2.0.0-py3-none-any.whl.metadata (3.5 kB)\n",
            "Requirement already satisfied: numpy>=1.18.0 in /usr/local/lib/python3.10/dist-packages (from shimmy) (1.26.4)\n",
            "Collecting gymnasium>=1.0.0a1 (from shimmy)\n",
            "  Downloading gymnasium-1.0.0-py3-none-any.whl.metadata (9.5 kB)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from gymnasium>=1.0.0a1->shimmy) (3.1.0)\n",
            "Requirement already satisfied: typing-extensions>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from gymnasium>=1.0.0a1->shimmy) (4.12.2)\n",
            "Requirement already satisfied: farama-notifications>=0.0.1 in /usr/local/lib/python3.10/dist-packages (from gymnasium>=1.0.0a1->shimmy) (0.0.4)\n",
            "Downloading Shimmy-2.0.0-py3-none-any.whl (30 kB)\n",
            "Downloading gymnasium-1.0.0-py3-none-any.whl (958 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m958.1/958.1 kB\u001b[0m \u001b[31m16.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: gymnasium, shimmy\n",
            "  Attempting uninstall: gymnasium\n",
            "    Found existing installation: gymnasium 0.29.1\n",
            "    Uninstalling gymnasium-0.29.1:\n",
            "      Successfully uninstalled gymnasium-0.29.1\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "stable-baselines3 2.3.2 requires gymnasium<0.30,>=0.28.1, but you have gymnasium 1.0.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed gymnasium-1.0.0 shimmy-2.0.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "gymnasium"
                ]
              },
              "id": "f645efe28d754803858e08d90557b715"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import gym\n",
        "from stable_baselines3 import PPO\n",
        "import skfuzzy as fuzz\n",
        "from skfuzzy import control as ctrl\n",
        "\n",
        "# 1. Define a Battery Simulation Environment (common for PPO and Fuzzy Logic)\n",
        "class BatteryEnv(gym.Env):\n",
        "    def __init__(self):\n",
        "        super(BatteryEnv, self).__init__()\n",
        "        self.action_space = gym.spaces.Box(low=0, high=1, shape=(1,), dtype=np.float32)\n",
        "        self.observation_space = gym.spaces.Box(low=0, high=100, shape=(2,), dtype=np.float32)\n",
        "        self.state = [50, 80]  # Initialize with [temperature, charge]\n",
        "\n",
        "    def reset(self):\n",
        "        self.state = [50, 80]\n",
        "        return np.array(self.state, dtype=np.float32)\n",
        "\n",
        "    def step(self, action):\n",
        "        temp, charge = self.state\n",
        "        cooling_power = action[0]\n",
        "        temp -= cooling_power * 5  # Cooling reduces temperature\n",
        "        charge -= 0.1 * (1 - cooling_power)\n",
        "        self.state = [temp, charge]\n",
        "        reward = -abs(temp - 40) - abs(charge - 80)\n",
        "        done = temp < 20 or temp > 80 or charge < 10\n",
        "        return np.array(self.state, dtype=np.float32), reward, done, {}\n",
        "\n",
        "# Initialize the environment\n",
        "env = BatteryEnv()\n",
        "\n",
        "# 2. Train PPO on the Environment\n",
        "print(\"Training PPO...\")\n",
        "model = PPO(\"MlpPolicy\", env, verbose=1)\n",
        "model.learn(total_timesteps=5000)\n",
        "\n",
        "# 3. Define the Fuzzy Logic System\n",
        "temperature = ctrl.Antecedent(np.arange(0, 101, 1), 'temperature')\n",
        "charge = ctrl.Antecedent(np.arange(0, 101, 1), 'charge')\n",
        "cooling_power = ctrl.Consequent(np.arange(0, 101, 1), 'cooling_power')\n",
        "\n",
        "temperature['low'] = fuzz.trimf(temperature.universe, [0, 0, 40])\n",
        "temperature['medium'] = fuzz.trimf(temperature.universe, [30, 50, 70])\n",
        "temperature['high'] = fuzz.trimf(temperature.universe, [60, 100, 100])\n",
        "charge['low'] = fuzz.trimf(charge.universe, [0, 0, 50])\n",
        "charge['medium'] = fuzz.trimf(charge.universe, [30, 50, 80])\n",
        "charge['high'] = fuzz.trimf(charge.universe, [70, 100, 100])\n",
        "cooling_power['low'] = fuzz.trimf(cooling_power.universe, [0, 0, 50])\n",
        "cooling_power['medium'] = fuzz.trimf(cooling_power.universe, [25, 50, 75])\n",
        "cooling_power['high'] = fuzz.trimf(cooling_power.universe, [50, 100, 100])\n",
        "\n",
        "rule1 = ctrl.Rule(temperature['high'] & charge['high'], cooling_power['high'])\n",
        "rule2 = ctrl.Rule(temperature['medium'] & charge['high'], cooling_power['medium'])\n",
        "rule3 = ctrl.Rule(temperature['low'] | charge['low'], cooling_power['low'])\n",
        "\n",
        "cooling_ctrl = ctrl.ControlSystem([rule1, rule2, rule3])\n",
        "cooling_sim = ctrl.ControlSystemSimulation(cooling_ctrl)\n",
        "\n",
        "# 4. Test and Compare PPO and Fuzzy Logic\n",
        "def test_policy(policy, env, policy_type=\"PPO\"):\n",
        "    obs = env.reset()\n",
        "    total_reward = 0\n",
        "    steps = 0\n",
        "    while True:\n",
        "        if policy_type == \"PPO\":\n",
        "            action, _ = policy.predict(obs, deterministic=True)\n",
        "        elif policy_type == \"Fuzzy\":\n",
        "            temp, charge = obs\n",
        "            cooling_sim.input['temperature'] = temp\n",
        "            cooling_sim.input['charge'] = charge\n",
        "            cooling_sim.compute()\n",
        "            action = [cooling_sim.output['cooling_power'] / 100.0]  # Normalize to 0-1\n",
        "        obs, reward, done, _ = env.step(action)\n",
        "        total_reward += reward\n",
        "        steps += 1\n",
        "        if done or steps > 100:  # Limit to 100 steps to avoid infinite loops\n",
        "            break\n",
        "    return total_reward\n",
        "\n",
        "# Run tests\n",
        "ppo_reward = test_policy(model, env, policy_type=\"PPO\")\n",
        "fuzzy_reward = test_policy(None, env, policy_type=\"Fuzzy\")\n",
        "\n",
        "print(\"\\nResults:\")\n",
        "print(f\"PPO Total Reward: {ppo_reward}\")\n",
        "print(f\"Fuzzy Logic Total Reward: {fuzzy_reward}\")\n",
        "\n",
        "# Comparison based on average reward and overall performance\n",
        "if ppo_reward > fuzzy_reward:\n",
        "    print(\"PPO outperformed Fuzzy Logic for this battery optimization task.\")\n",
        "else:\n",
        "    print(\"Fuzzy Logic outperformed PPO for this battery optimization task.\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "szuyHVS8DSkI",
        "outputId": "183f8a7c-499d-4354-a89e-444da11d4785"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training PPO...\n",
            "Using cpu device\n",
            "Wrapping the env with a `Monitor` wrapper\n",
            "Wrapping the env in a DummyVecEnv.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/stable_baselines3/common/vec_env/patch_gym.py:49: UserWarning: You provided an OpenAI Gym environment. We strongly recommend transitioning to Gymnasium environments. Stable-Baselines3 is automatically wrapping your environments in a compatibility layer, which could potentially cause issues.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 20.7     |\n",
            "|    ep_rew_mean     | -206     |\n",
            "| time/              |          |\n",
            "|    fps             | 1134     |\n",
            "|    iterations      | 1        |\n",
            "|    time_elapsed    | 1        |\n",
            "|    total_timesteps | 2048     |\n",
            "---------------------------------\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 17.9        |\n",
            "|    ep_rew_mean          | -169        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 843         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 4           |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.005569823 |\n",
            "|    clip_fraction        | 0.084       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.41       |\n",
            "|    explained_variance   | 0.00396     |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 3.43e+03    |\n",
            "|    n_updates            | 10          |\n",
            "|    policy_gradient_loss | -0.0155     |\n",
            "|    std                  | 0.982       |\n",
            "|    value_loss           | 8.08e+03    |\n",
            "-----------------------------------------\n",
            "------------------------------------------\n",
            "| rollout/                |              |\n",
            "|    ep_len_mean          | 15.6         |\n",
            "|    ep_rew_mean          | -146         |\n",
            "| time/                   |              |\n",
            "|    fps                  | 782          |\n",
            "|    iterations           | 3            |\n",
            "|    time_elapsed         | 7            |\n",
            "|    total_timesteps      | 6144         |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0069914893 |\n",
            "|    clip_fraction        | 0.125        |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -1.39        |\n",
            "|    explained_variance   | 0.000601     |\n",
            "|    learning_rate        | 0.0003       |\n",
            "|    loss                 | 2.64e+03     |\n",
            "|    n_updates            | 20           |\n",
            "|    policy_gradient_loss | -0.0179      |\n",
            "|    std                  | 0.962        |\n",
            "|    value_loss           | 5.74e+03     |\n",
            "------------------------------------------\n",
            "\n",
            "Results:\n",
            "PPO Total Reward: -164.61741319894853\n",
            "Fuzzy Logic Total Reward: -218.6597442236062\n",
            "PPO outperformed Fuzzy Logic for this battery optimization task.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import gym\n",
        "from stable_baselines3 import PPO\n",
        "import skfuzzy as fuzz\n",
        "from skfuzzy import control as ctrl\n",
        "\n",
        "# 1. Define Battery Consumption Environment\n",
        "class BatteryEnv(gym.Env):\n",
        "    def __init__(self):\n",
        "        super(BatteryEnv, self).__init__()\n",
        "        # Action: Power consumption level, continuous between 0 (low) and 1 (high)\n",
        "        self.action_space = gym.spaces.Box(low=0, high=1, shape=(1,), dtype=np.float32)\n",
        "        # Observation: Battery level, speed, and incline angle\n",
        "        self.observation_space = gym.spaces.Box(low=np.array([0, 0, -10]), high=np.array([100, 120, 10]), dtype=np.float32)\n",
        "        self.state = [100, 50, 0]  # Initial state: [battery level, speed, incline angle]\n",
        "\n",
        "    def reset(self):\n",
        "        self.state = [100, 50, 0]  # Reset to full battery, moderate speed, flat road\n",
        "        return np.array(self.state, dtype=np.float32)\n",
        "\n",
        "    def step(self, action):\n",
        "        battery, speed, incline = self.state\n",
        "        power_consumption = action[0]\n",
        "\n",
        "        # Battery drain: Increased by speed and incline\n",
        "        battery -= power_consumption * (1 + speed / 100 + incline / 10)\n",
        "        battery = max(battery, 0)  # Ensure battery does not go below 0\n",
        "\n",
        "        # Simulate changes in speed and incline\n",
        "        speed = np.clip(speed + np.random.uniform(-5, 5), 0, 120)\n",
        "        incline = np.clip(incline + np.random.uniform(-2, 2), -10, 10)\n",
        "\n",
        "        self.state = [battery, speed, incline]\n",
        "        reward = -power_consumption * (1 + speed / 100 + incline / 10)  # Penalize high consumption\n",
        "        done = battery <= 0\n",
        "        return np.array(self.state, dtype=np.float32), reward, done, {}\n",
        "\n",
        "# Initialize the environment\n",
        "env = BatteryEnv()\n",
        "\n",
        "# 2. Train PPO on the Environment\n",
        "print(\"Training PPO...\")\n",
        "model = PPO(\"MlpPolicy\", env, verbose=1)\n",
        "model.learn(total_timesteps=5000)\n",
        "\n",
        "# 3. Define the Fuzzy Logic System\n",
        "battery_level = ctrl.Antecedent(np.arange(0, 101, 1), 'battery_level')\n",
        "speed = ctrl.Antecedent(np.arange(0, 121, 1), 'speed')\n",
        "incline = ctrl.Antecedent(np.arange(-10, 11, 1), 'incline')\n",
        "power = ctrl.Consequent(np.arange(0, 101, 1), 'power')\n",
        "\n",
        "# Define fuzzy sets\n",
        "battery_level['low'] = fuzz.trimf(battery_level.universe, [0, 0, 40])\n",
        "battery_level['medium'] = fuzz.trimf(battery_level.universe, [30, 50, 70])\n",
        "battery_level['high'] = fuzz.trimf(battery_level.universe, [60, 100, 100])\n",
        "\n",
        "speed['slow'] = fuzz.trimf(speed.universe, [0, 0, 60])\n",
        "speed['medium'] = fuzz.trimf(speed.universe, [40, 60, 80])\n",
        "speed['fast'] = fuzz.trimf(speed.universe, [70, 120, 120])\n",
        "\n",
        "incline['downhill'] = fuzz.trimf(incline.universe, [-10, -10, 0])\n",
        "incline['flat'] = fuzz.trimf(incline.universe, [-2, 0, 2])\n",
        "incline['uphill'] = fuzz.trimf(incline.universe, [0, 10, 10])\n",
        "\n",
        "power['low'] = fuzz.trimf(power.universe, [0, 0, 50])\n",
        "power['medium'] = fuzz.trimf(power.universe, [25, 50, 75])\n",
        "power['high'] = fuzz.trimf(power.universe, [50, 100, 100])\n",
        "\n",
        "# Define rules\n",
        "rule1 = ctrl.Rule(battery_level['high'] & incline['downhill'], power['low'])\n",
        "rule2 = ctrl.Rule(battery_level['medium'] & incline['flat'], power['medium'])\n",
        "rule3 = ctrl.Rule(battery_level['low'] & incline['uphill'], power['high'])\n",
        "rule4 = ctrl.Rule(speed['fast'] & incline['uphill'], power['high'])\n",
        "rule5 = ctrl.Rule(speed['slow'] & incline['downhill'], power['low'])\n",
        "\n",
        "# Create the fuzzy control system\n",
        "power_ctrl = ctrl.ControlSystem([rule1, rule2, rule3, rule4, rule5])\n",
        "power_sim = ctrl.ControlSystemSimulation(power_ctrl)\n",
        "\n",
        "# 4. Test and Compare PPO and Fuzzy Logic\n",
        "def test_policy(policy, env, policy_type=\"PPO\"):\n",
        "    obs = env.reset()\n",
        "    total_reward = 0\n",
        "    steps = 0\n",
        "    while True:\n",
        "        if policy_type == \"PPO\":\n",
        "            action, _ = policy.predict(obs, deterministic=True)\n",
        "        elif policy_type == \"Fuzzy\":\n",
        "            battery, speed, incline = obs\n",
        "            power_sim.input['battery_level'] = battery\n",
        "            power_sim.input['speed'] = speed\n",
        "            power_sim.input['incline'] = incline\n",
        "            power_sim.compute()  # Compute the fuzzy logic output\n",
        "            action = [power_sim.output['power'] / 100.0]  # Normalize to 0-1 range\n",
        "\n",
        "        obs, reward, done, _ = env.step(action)\n",
        "        total_reward += reward\n",
        "        steps += 1\n",
        "        if done or steps > 100:  # Limit to 100 steps\n",
        "            break\n",
        "    return total_reward\n",
        "\n",
        "# Run tests\n",
        "ppo_reward = test_policy(model, env, policy_type=\"PPO\")\n",
        "fuzzy_reward = test_policy(None, env, policy_type=\"Fuzzy\")\n",
        "\n",
        "print(\"\\nResults:\")\n",
        "print(f\"PPO Total Reward: {ppo_reward}\")\n",
        "print(f\"Fuzzy Logic Total Reward: {fuzzy_reward}\")\n",
        "\n",
        "# Display which approach performed better\n",
        "if ppo_reward > fuzzy_reward:\n",
        "    print(\"PPO outperformed Fuzzy Logic for battery consumption optimization.\")\n",
        "else:\n",
        "    print(\"Fuzzy Logic outperformed PPO for battery consumption optimization.\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "-v6cJlkzEO2P",
        "outputId": "ed0cf6e2-ac8a-4f22-ad46-786064a8b220"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n",
            "/usr/local/lib/python3.10/dist-packages/gym/spaces/box.py:128: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
            "  logger.warn(f\"Box bound precision lowered by casting to {self.dtype}\")\n",
            "/usr/local/lib/python3.10/dist-packages/stable_baselines3/common/vec_env/patch_gym.py:49: UserWarning: You provided an OpenAI Gym environment. We strongly recommend transitioning to Gymnasium environments. Stable-Baselines3 is automatically wrapping your environments in a compatibility layer, which could potentially cause issues.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training PPO...\n",
            "Using cpu device\n",
            "Wrapping the env with a `Monitor` wrapper\n",
            "Wrapping the env in a DummyVecEnv.\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 208      |\n",
            "|    ep_rew_mean     | -100     |\n",
            "| time/              |          |\n",
            "|    fps             | 1237     |\n",
            "|    iterations      | 1        |\n",
            "|    time_elapsed    | 1        |\n",
            "|    total_timesteps | 2048     |\n",
            "---------------------------------\n",
            "----------------------------------------\n",
            "| rollout/                |            |\n",
            "|    ep_len_mean          | 254        |\n",
            "|    ep_rew_mean          | -100       |\n",
            "| time/                   |            |\n",
            "|    fps                  | 829        |\n",
            "|    iterations           | 2          |\n",
            "|    time_elapsed         | 4          |\n",
            "|    total_timesteps      | 4096       |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.01195502 |\n",
            "|    clip_fraction        | 0.129      |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -1.4       |\n",
            "|    explained_variance   | -0.0244    |\n",
            "|    learning_rate        | 0.0003     |\n",
            "|    loss                 | 3.3        |\n",
            "|    n_updates            | 10         |\n",
            "|    policy_gradient_loss | -0.013     |\n",
            "|    std                  | 0.962      |\n",
            "|    value_loss           | 15.2       |\n",
            "----------------------------------------\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 265         |\n",
            "|    ep_rew_mean          | -101        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 707         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 8           |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.007905252 |\n",
            "|    clip_fraction        | 0.0691      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.37       |\n",
            "|    explained_variance   | 0.589       |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 2.47        |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.00767    |\n",
            "|    std                  | 0.941       |\n",
            "|    value_loss           | 9.93        |\n",
            "-----------------------------------------\n",
            "OrderedDict()\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "UnboundLocalError",
          "evalue": "local variable 'action' referenced before assignment",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mUnboundLocalError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-11-537a7fc1798b>\u001b[0m in \u001b[0;36m<cell line: 106>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    104\u001b[0m \u001b[0;31m# Run tests\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m \u001b[0mppo_reward\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtest_policy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpolicy_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"PPO\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 106\u001b[0;31m \u001b[0mfuzzy_reward\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtest_policy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpolicy_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"Fuzzy\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    107\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    108\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\\nResults:\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-11-537a7fc1798b>\u001b[0m in \u001b[0;36mtest_policy\u001b[0;34m(policy, env, policy_type)\u001b[0m\n\u001b[1;32m     94\u001b[0m             \u001b[0;31m# action = [power_sim.output['power'] / 100.0]  # Normalize to 0-1 range\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 96\u001b[0;31m         \u001b[0mobs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     97\u001b[0m         \u001b[0mtotal_reward\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m         \u001b[0msteps\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mUnboundLocalError\u001b[0m: local variable 'action' referenced before assignment"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import gym\n",
        "from stable_baselines3 import PPO\n",
        "import random\n",
        "import math\n",
        "\n",
        "# 1. Define Battery Consumption Environment\n",
        "class BatteryEnv(gym.Env):\n",
        "    def __init__(self):\n",
        "        super(BatteryEnv, self).__init__()\n",
        "        self.action_space = gym.spaces.Box(low=0, high=1, shape=(1,), dtype=np.float32)\n",
        "        self.observation_space = gym.spaces.Box(low=np.array([0, 0, -10]), high=np.array([100, 120, 10]), dtype=np.float32)\n",
        "        self.state = [100, 50, 0]  # Initial state: [battery level, speed, incline]\n",
        "\n",
        "    def reset(self):\n",
        "        self.state = [100, 50, 0]  # Reset to full battery, moderate speed, flat road\n",
        "        return np.array(self.state, dtype=np.float32)\n",
        "\n",
        "    def step(self, action):\n",
        "        battery, speed, incline = self.state\n",
        "        power_consumption = action[0]\n",
        "\n",
        "        battery -= power_consumption * (1 + speed / 100 + incline / 10)\n",
        "        battery = max(battery, 0)  # Ensure battery does not go below 0\n",
        "\n",
        "        speed = np.clip(speed + np.random.uniform(-5, 5), 0, 120)\n",
        "        incline = np.clip(incline + np.random.uniform(-2, 2), -10, 10)\n",
        "\n",
        "        self.state = [battery, speed, incline]\n",
        "        reward = -power_consumption * (1 + speed / 100 + incline / 10)\n",
        "        done = battery <= 0\n",
        "        return np.array(self.state, dtype=np.float32), reward, done, {}\n",
        "\n",
        "# Initialize the environment\n",
        "env = BatteryEnv()\n",
        "\n",
        "# 2. Train PPO on the Environment\n",
        "print(\"Training PPO...\")\n",
        "model = PPO(\"MlpPolicy\", env, verbose=1)\n",
        "model.learn(total_timesteps=5000)\n",
        "\n",
        "# 3. Implement Sidewinder Snake Algorithm\n",
        "class SidewinderSnake:\n",
        "    def __init__(self, env):\n",
        "        self.env = env\n",
        "\n",
        "    def choose_action(self, state):\n",
        "        # A simple heuristic: decide the action based on battery level and incline\n",
        "        battery, speed, incline = state\n",
        "\n",
        "        if battery > 80:\n",
        "            action = random.uniform(0.1, 0.3)  # Low consumption\n",
        "        elif battery > 40:\n",
        "            action = random.uniform(0.3, 0.7)  # Medium consumption\n",
        "        else:\n",
        "            action = random.uniform(0.7, 1.0)  # High consumption\n",
        "\n",
        "        # \"Snake\" behavior: introduce a random component to simulate exploration\n",
        "        action += random.uniform(-0.05, 0.05)\n",
        "        action = np.clip(action, 0, 1)\n",
        "\n",
        "        return [action]\n",
        "\n",
        "    def test_policy(self, env):\n",
        "        state = env.reset()\n",
        "        total_reward = 0\n",
        "        done = False\n",
        "        while not done:\n",
        "            action = self.choose_action(state)\n",
        "            state, reward, done, _ = env.step(action)\n",
        "            total_reward += reward\n",
        "        return total_reward\n",
        "\n",
        "# 4. Test function for both PPO and Sidewinder Snake Algorithm\n",
        "def test_policy(model, env, policy_type=\"PPO\"):\n",
        "    if policy_type == \"PPO\":\n",
        "        state = env.reset()\n",
        "        total_reward = 0\n",
        "        done = False\n",
        "        while not done:\n",
        "            action, _states = model.predict(state, deterministic=True)\n",
        "            state, reward, done, _ = env.step(action)\n",
        "            total_reward += reward\n",
        "        return total_reward\n",
        "    elif policy_type == \"Fuzzy\":\n",
        "        snake_algorithm = SidewinderSnake(env)\n",
        "        return snake_algorithm.test_policy(env)\n",
        "\n",
        "# 5. Compare PPO and Sidewinder Snake Algorithm Performance\n",
        "def compare_algorithms(env):\n",
        "    # Test PPO\n",
        "    ppo_reward = test_policy(model, env, policy_type=\"PPO\")\n",
        "\n",
        "    # Test Sidewinder Snake Algorithm\n",
        "    snake_algorithm = SidewinderSnake(env)\n",
        "    sidewinder_reward = snake_algorithm.test_policy(env)\n",
        "\n",
        "    # Display Results\n",
        "    print(\"\\nResults:\")\n",
        "    print(f\"PPO Total Reward: {ppo_reward}\")\n",
        "    print(f\"Sidewinder Snake Algorithm Total Reward: {sidewinder_reward}\")\n",
        "\n",
        "    if ppo_reward > sidewinder_reward:\n",
        "        print(\"PPO outperformed Sidewinder Snake Algorithm.\")\n",
        "    else:\n",
        "        print(\"Sidewinder Snake Algorithm outperformed PPO.\")\n",
        "\n",
        "# Run comparison\n",
        "compare_algorithms(env)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "QcFiozgsGA6Q",
        "outputId": "a711f65e-3a21-44f6-aa3e-df596f8c1959"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training PPO...\n",
            "Using cpu device\n",
            "Wrapping the env with a `Monitor` wrapper\n",
            "Wrapping the env in a DummyVecEnv.\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 208      |\n",
            "|    ep_rew_mean     | -101     |\n",
            "| time/              |          |\n",
            "|    fps             | 888      |\n",
            "|    iterations      | 1        |\n",
            "|    time_elapsed    | 2        |\n",
            "|    total_timesteps | 2048     |\n",
            "---------------------------------\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 225         |\n",
            "|    ep_rew_mean          | -101        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 760         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 5           |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.014373256 |\n",
            "|    clip_fraction        | 0.182       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.41       |\n",
            "|    explained_variance   | -0.00195    |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 3.16        |\n",
            "|    n_updates            | 10          |\n",
            "|    policy_gradient_loss | -0.0188     |\n",
            "|    std                  | 0.984       |\n",
            "|    value_loss           | 13.5        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 251         |\n",
            "|    ep_rew_mean          | -101        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 726         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 8           |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.010698045 |\n",
            "|    clip_fraction        | 0.115       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.39       |\n",
            "|    explained_variance   | 0.488       |\n",
            "|    learning_rate        | 0.0003      |\n",
            "|    loss                 | 2.51        |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.0135     |\n",
            "|    std                  | 0.962       |\n",
            "|    value_loss           | 11.2        |\n",
            "-----------------------------------------\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-12-04ad22b7fcdf>\u001b[0m in \u001b[0;36m<cell line: 109>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    107\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    108\u001b[0m \u001b[0;31m# Run comparison\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 109\u001b[0;31m \u001b[0mcompare_algorithms\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-12-04ad22b7fcdf>\u001b[0m in \u001b[0;36mcompare_algorithms\u001b[0;34m(env)\u001b[0m\n\u001b[1;32m     90\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mcompare_algorithms\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m     \u001b[0;31m# Test PPO\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m     \u001b[0mppo_reward\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtest_policy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpolicy_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"PPO\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m     \u001b[0;31m# Test Sidewinder Snake Algorithm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-12-04ad22b7fcdf>\u001b[0m in \u001b[0;36mtest_policy\u001b[0;34m(model, env, policy_type)\u001b[0m\n\u001b[1;32m     79\u001b[0m         \u001b[0mdone\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 81\u001b[0;31m             \u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_states\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     82\u001b[0m             \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     83\u001b[0m             \u001b[0mtotal_reward\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/stable_baselines3/common/base_class.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, observation, state, episode_start, deterministic)\u001b[0m\n\u001b[1;32m    554\u001b[0m             \u001b[0;34m(\u001b[0m\u001b[0mused\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrecurrent\u001b[0m \u001b[0mpolicies\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    555\u001b[0m         \"\"\"\n\u001b[0;32m--> 556\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpolicy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobservation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepisode_start\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    557\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    558\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mset_random_seed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/stable_baselines3/common/policies.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, observation, state, episode_start, deterministic)\u001b[0m\n\u001b[1;32m    366\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    367\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 368\u001b[0;31m             \u001b[0mactions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobs_tensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdeterministic\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    369\u001b[0m         \u001b[0;31m# Convert to numpy, and reshape to the original action shape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    370\u001b[0m         \u001b[0mactions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mactions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[misc]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/stable_baselines3/common/policies.py\u001b[0m in \u001b[0;36m_predict\u001b[0;34m(self, observation, deterministic)\u001b[0m\n\u001b[1;32m    715\u001b[0m         \u001b[0;34m:\u001b[0m\u001b[0;32mreturn\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTaken\u001b[0m \u001b[0maction\u001b[0m \u001b[0maccording\u001b[0m \u001b[0mto\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mpolicy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    716\u001b[0m         \"\"\"\n\u001b[0;32m--> 717\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_distribution\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobservation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_actions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdeterministic\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdeterministic\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    718\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    719\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mevaluate_actions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mPyTorchObs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mactions\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTuple\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/stable_baselines3/common/policies.py\u001b[0m in \u001b[0;36mget_distribution\u001b[0;34m(self, obs)\u001b[0m\n\u001b[1;32m    750\u001b[0m         \u001b[0mfeatures\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextract_features\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpi_features_extractor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    751\u001b[0m         \u001b[0mlatent_pi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmlp_extractor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward_actor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 752\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_action_dist_from_latent\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlatent_pi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    753\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    754\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpredict_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mPyTorchObs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/stable_baselines3/common/policies.py\u001b[0m in \u001b[0;36m_get_action_dist_from_latent\u001b[0;34m(self, latent_pi)\u001b[0m\n\u001b[1;32m    692\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    693\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_dist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDiagGaussianDistribution\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 694\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_dist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mproba_distribution\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmean_actions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog_std\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    695\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_dist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mCategoricalDistribution\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    696\u001b[0m             \u001b[0;31m# Here mean_actions are the logits before the softmax\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/stable_baselines3/common/distributions.py\u001b[0m in \u001b[0;36mproba_distribution\u001b[0;34m(self, mean_actions, log_std)\u001b[0m\n\u001b[1;32m    162\u001b[0m         \"\"\"\n\u001b[1;32m    163\u001b[0m         \u001b[0maction_std\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mones_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmean_actions\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mlog_std\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 164\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdistribution\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNormal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmean_actions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction_std\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    165\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/distributions/normal.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, loc, scale, validate_args)\u001b[0m\n\u001b[1;32m     57\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m             \u001b[0mbatch_shape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 59\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_shape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_args\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     60\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mexpand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_shape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_instance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/distributions/distribution.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, batch_shape, event_shape, validate_args)\u001b[0m\n\u001b[1;32m     68\u001b[0m                 \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m                 \u001b[0mvalid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconstraint\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcheck\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mvalid\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     71\u001b[0m                     raise ValueError(\n\u001b[1;32m     72\u001b[0m                         \u001b[0;34mf\"Expected parameter {param} \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import gym\n",
        "from stable_baselines3 import PPO\n",
        "import random\n",
        "\n",
        "# 1. Define Battery Consumption Environment\n",
        "class BatteryEnv(gym.Env):\n",
        "    def __init__(self):\n",
        "        super(BatteryEnv, self).__init__()\n",
        "        self.action_space = gym.spaces.Box(low=0, high=1, shape=(1,), dtype=np.float32)  # Power consumption action space\n",
        "        self.observation_space = gym.spaces.Box(low=np.array([0, 0, -10]), high=np.array([100, 120, 10]), dtype=np.float32)  # Battery, speed, incline\n",
        "        self.state = [100, 50, 0]  # Initial state: [battery level, speed, incline]\n",
        "\n",
        "    def reset(self):\n",
        "        self.state = [100, 50, 0]  # Reset to full battery, moderate speed, flat road\n",
        "        return np.array(self.state, dtype=np.float32)\n",
        "\n",
        "    def step(self, action):\n",
        "        battery, speed, incline = self.state\n",
        "        power_consumption = action[0] * 100  # Scale power consumption to a reasonable range (e.g., 0-100)\n",
        "\n",
        "        # Update battery based on power consumption and the effect of speed and incline\n",
        "        battery -= power_consumption * (1 + speed / 100 + incline / 10)\n",
        "        battery = max(battery, 0)  # Ensure battery does not go below 0\n",
        "\n",
        "        # Random changes to speed and incline\n",
        "        speed = np.clip(speed + np.random.uniform(-5, 5), 0, 120)\n",
        "        incline = np.clip(incline + np.random.uniform(-2, 2), -10, 10)\n",
        "\n",
        "        self.state = [battery, speed, incline]\n",
        "\n",
        "        # Reward is negative because we want to minimize battery usage\n",
        "        reward = -power_consumption * (1 + speed / 100 + incline / 10)\n",
        "\n",
        "        # Episode ends when battery is drained\n",
        "        done = battery <= 0\n",
        "        return np.array(self.state, dtype=np.float32), reward, done, {}\n",
        "\n",
        "# Initialize the environment\n",
        "env = BatteryEnv()\n",
        "\n",
        "# 2. Train PPO on the Environment\n",
        "print(\"Training PPO...\")\n",
        "model = PPO(\"MlpPolicy\", env, verbose=1)\n",
        "model.learn(total_timesteps=5000)\n",
        "\n",
        "# 3. Test function for both PPO and Normal Agent\n",
        "def test_policy(model, env, policy_type=\"PPO\"):\n",
        "    state = env.reset()\n",
        "    total_reward = 0\n",
        "    done = False\n",
        "    c=0\n",
        "    while not done:\n",
        "        if policy_type == \"PPO\":\n",
        "            # PPO uses its learned policy to choose the action\n",
        "            action, _states = model.predict(state, deterministic=True)\n",
        "        elif policy_type == \"Normal\":\n",
        "            # Normal agent takes random actions\n",
        "            action = np.random.uniform(0, 1, size=(1,))\n",
        "        c+=1\n",
        "        state, reward, done, _ = env.step(action)\n",
        "        total_reward += reward\n",
        "        if done or c>100:\n",
        "          break\n",
        "    return total_reward\n",
        "\n",
        "# 4. Compare PPO and Normal Agent Performance\n",
        "def compare_algorithms(env):\n",
        "    # Test PPO\n",
        "    ppo_reward = test_policy(model, env, policy_type=\"PPO\")\n",
        "\n",
        "    # Test Normal Agent\n",
        "    normal_reward = test_policy(None, env, policy_type=\"Normal\")\n",
        "\n",
        "    # Display Results\n",
        "    print(\"\\nResults:\")\n",
        "    print(f\"PPO Total Reward: {ppo_reward}\")\n",
        "    print(f\"Normal Agent Total Reward: {normal_reward}\")\n",
        "\n",
        "    if ppo_reward > normal_reward:\n",
        "        print(\"PPO outperformed Normal Agent.\")\n",
        "    else:\n",
        "        print(\"Normal Agent outperformed PPO.\")\n",
        "\n",
        "# Run comparison\n",
        "compare_algorithms(env)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uMD3PdPoHCfB",
        "outputId": "78afc4cb-3dd3-4c50-81f2-b98a69c6241e"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
            "  and should_run_async(code)\n",
            "/usr/local/lib/python3.10/dist-packages/gym/spaces/box.py:128: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
            "  logger.warn(f\"Box bound precision lowered by casting to {self.dtype}\")\n",
            "/usr/local/lib/python3.10/dist-packages/stable_baselines3/common/vec_env/patch_gym.py:49: UserWarning: You provided an OpenAI Gym environment. We strongly recommend transitioning to Gymnasium environments. Stable-Baselines3 is automatically wrapping your environments in a compatibility layer, which could potentially cause issues.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training PPO...\n",
            "Using cpu device\n",
            "Wrapping the env with a `Monitor` wrapper\n",
            "Wrapping the env in a DummyVecEnv.\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 2.87     |\n",
            "|    ep_rew_mean     | -157     |\n",
            "| time/              |          |\n",
            "|    fps             | 1224     |\n",
            "|    iterations      | 1        |\n",
            "|    time_elapsed    | 1        |\n",
            "|    total_timesteps | 2048     |\n",
            "---------------------------------\n",
            "----------------------------------------\n",
            "| rollout/                |            |\n",
            "|    ep_len_mean          | 4.53       |\n",
            "|    ep_rew_mean          | -158       |\n",
            "| time/                   |            |\n",
            "|    fps                  | 862        |\n",
            "|    iterations           | 2          |\n",
            "|    time_elapsed         | 4          |\n",
            "|    total_timesteps      | 4096       |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.01892561 |\n",
            "|    clip_fraction        | 0.214      |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -1.39      |\n",
            "|    explained_variance   | -0.00389   |\n",
            "|    learning_rate        | 0.0003     |\n",
            "|    loss                 | 7.85e+03   |\n",
            "|    n_updates            | 10         |\n",
            "|    policy_gradient_loss | -0.0236    |\n",
            "|    std                  | 0.948      |\n",
            "|    value_loss           | 1.62e+04   |\n",
            "----------------------------------------\n",
            "----------------------------------------\n",
            "| rollout/                |            |\n",
            "|    ep_len_mean          | 5.1        |\n",
            "|    ep_rew_mean          | -156       |\n",
            "| time/                   |            |\n",
            "|    fps                  | 780        |\n",
            "|    iterations           | 3          |\n",
            "|    time_elapsed         | 7          |\n",
            "|    total_timesteps      | 6144       |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.01596106 |\n",
            "|    clip_fraction        | 0.192      |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -1.34      |\n",
            "|    explained_variance   | 0.00242    |\n",
            "|    learning_rate        | 0.0003     |\n",
            "|    loss                 | 6.77e+03   |\n",
            "|    n_updates            | 20         |\n",
            "|    policy_gradient_loss | -0.0221    |\n",
            "|    std                  | 0.904      |\n",
            "|    value_loss           | 1.35e+04   |\n",
            "----------------------------------------\n",
            "\n",
            "Results:\n",
            "PPO Total Reward: 0.0\n",
            "Normal Agent Total Reward: -113.49948210490172\n",
            "PPO outperformed Normal Agent.\n"
          ]
        }
      ]
    }
  ]
}