colabs/NanoGPT_Quickstart.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/klei22/nanoGPT/blob/master/NanoGPT_Quickstart.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "<div class=\"mrkdown-google-sans\">\n",
        "\n",
        "# **GPU Quickstart**\n",
        "\n",
        "</div>"
      ],
      "metadata": {
        "id": "swIfEfkhpCSZ"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7Qfh9x6myI9g"
      },
      "source": [
        "<div class=\"mrkdown-google-sans\">\n",
        "\n",
        "### **Install NanoGPT GPU Dependencies**\n",
        "\n",
        "</div>\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd\n",
        "!rm -rf nanoGPT_gpu\n",
        "!git clone https://github.com/ReaLLMASIC/nanoGPT.git nanoGPT_gpu\n",
        "%cd nanoGPT_gpu\n",
        "\n",
        "# check branch info\n",
        "!echo \"Cloned repository\"\n",
        "!git branch\n",
        "\n",
        "!ls\n",
        "\n",
        "!pip install --upgrade pip\n",
        "!pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118\n",
        "!pip install numpy transformers datasets tiktoken wandb tqdm tensorboard\n"
      ],
      "metadata": {
        "id": "z9TWJn8PpJLg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "103dcd4f-8166-45dc-b4c9-61f9f65c9f13"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/root\n",
            "Cloning into 'nanoGPT_gpu'...\n",
            "remote: Enumerating objects: 1328, done.\u001b[K\n",
            "remote: Counting objects: 100% (20/20), done.\u001b[K\n",
            "remote: Compressing objects: 100% (14/14), done.\u001b[K\n",
            "remote: Total 1328 (delta 7), reused 15 (delta 6), pack-reused 1308\u001b[K\n",
            "Receiving objects: 100% (1328/1328), 2.79 MiB | 6.83 MiB/s, done.\n",
            "Resolving deltas: 100% (748/748), done.\n",
            "/root/nanoGPT_gpu\n",
            "Cloned repository\n",
            "* \u001b[32mmaster\u001b[m\n",
            "assets\t\t\t  install_llama_cpp_python.sh  sample.py\n",
            "bench.py\t\t  LICENSE\t\t       scaling_laws.ipynb\n",
            "config\t\t\t  model.py\t\t       start_tensorboard.sh\n",
            "Contributing_Features.md  modules\t\t       tests\n",
            "data\t\t\t  NanoGPT_Quickstart.ipynb     train.py\n",
            "data_augmentation\t  README.md\t\t       transformer_sizing.ipynb\n",
            "explorations\t\t  requirements_cpu.txt\t       variations\n",
            "HW\t\t\t  run_experiments.py\n",
            "Requirement already satisfied: pip in /usr/local/lib/python3.10/dist-packages (23.3.2)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://download.pytorch.org/whl/cu118\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.1.0+cu121)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (0.16.0+cu121)\n",
            "Requirement already satisfied: torchaudio in /usr/local/lib/python3.10/dist-packages (2.1.0+cu121)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.13.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch) (4.5.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.3)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2023.6.0)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch) (2.1.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from torchvision) (1.23.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from torchvision) (2.31.0)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision) (9.4.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (2.1.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision) (2023.11.17)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (1.23.5)\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.35.2)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (2.16.1)\n",
            "Requirement already satisfied: tiktoken in /usr/local/lib/python3.10/dist-packages (0.5.2)\n",
            "Requirement already satisfied: wandb in /usr/local/lib/python3.10/dist-packages (0.16.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (4.66.1)\n",
            "Requirement already satisfied: tensorboard in /usr/local/lib/python3.10/dist-packages (2.15.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.13.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.20.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.15.0)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.1)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (10.0.1)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n",
            "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.3.7)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets) (0.70.15)\n",
            "Requirement already satisfied: fsspec<=2023.10.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.1)\n",
            "Requirement already satisfied: Click!=8.0.0,>=7.1 in /usr/local/lib/python3.10/dist-packages (from wandb) (8.1.7)\n",
            "Requirement already satisfied: GitPython!=3.1.29,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (3.1.41)\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (5.9.5)\n",
            "Requirement already satisfied: sentry-sdk>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (1.39.2)\n",
            "Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (0.4.0)\n",
            "Requirement already satisfied: setproctitle in /usr/local/lib/python3.10/dist-packages (from wandb) (1.3.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb) (67.7.2)\n",
            "Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb) (1.4.4)\n",
            "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (3.20.3)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.10/dist-packages (from tensorboard) (1.4.0)\n",
            "Requirement already satisfied: grpcio>=1.48.2 in /usr/local/lib/python3.10/dist-packages (from tensorboard) (1.60.0)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.10/dist-packages (from tensorboard) (2.17.3)\n",
            "Requirement already satisfied: google-auth-oauthlib<2,>=0.5 in /usr/local/lib/python3.10/dist-packages (from tensorboard) (1.2.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard) (3.5.2)\n",
            "Requirement already satisfied: six>1.9 in /usr/local/lib/python3.10/dist-packages (from tensorboard) (1.16.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard) (0.7.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard) (3.0.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.2.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.4)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from GitPython!=3.1.29,>=1.0.0->wandb) (4.0.11)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard) (5.3.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard) (0.3.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard) (4.9)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from google-auth-oauthlib<2,>=0.5->tensorboard) (1.3.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (4.5.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.11.17)\n",
            "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.10/dist-packages (from werkzeug>=1.0.1->tensorboard) (2.1.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.3.post1)\n",
            "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.10/dist-packages (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb) (5.0.1)\n",
            "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard) (0.5.1)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<2,>=0.5->tensorboard) (3.2.2)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Za1OziLayNmh"
      },
      "source": [
        "<div class=\"mrkdown-google-sans\">\n",
        "\n",
        "### **Run GPU Training**\n",
        "\n",
        "</div>\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python3 data/shakespeare_char/prepare.py"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f-c73d8WyfIZ",
        "outputId": "5eb862f3-3b63-4e57-8ffd-6028e1191806"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "length of dataset in characters: 1,115,394\n",
            "all the unique characters: \n",
            " !$&',-.3:;?ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\n",
            "vocab size: 65\n",
            "train has 1,003,854 tokens\n",
            "val has 111,540 tokens\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python3 train.py --device=\"cuda\" --dtype=\"float16\" --max_iters=3500 --out_dir=\"out-shakespeare\""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "42aeaf2e-2d9d-4efb-a3d0-fe942ee0a4bf",
        "id": "GvvoJVMXQ6x5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "length of dataset in characters: 1,115,394\n",
            "all the unique characters: \n",
            " !$&',-.3:;?ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\n",
            "vocab size: 65\n",
            "train has 1,003,854 tokens\n",
            "val has 111,540 tokens\n",
            "2024-01-20 00:01:47.130106: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-01-20 00:01:47.130155: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-01-20 00:01:47.131508: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-01-20 00:01:47.138628: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-01-20 00:01:48.106963: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "\u001b[1m{\u001b[0m\n",
            "    \u001b[32m'block_size'\u001b[0m: \u001b[1;36m256\u001b[0m,\n",
            "    \u001b[32m'n_layer'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_head'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_embd'\u001b[0m: \u001b[1;36m384\u001b[0m,\n",
            "    \u001b[32m'dropout'\u001b[0m: \u001b[1;36m0.2\u001b[0m,\n",
            "    \u001b[32m'use_post_ln'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'layernorm_variant'\u001b[0m: \u001b[32m'rmsnorm'\u001b[0m,\n",
            "    \u001b[32m'bias'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'activation_variant'\u001b[0m: \u001b[32m'gelu'\u001b[0m,\n",
            "    \u001b[32m'use_rotary_embeddings'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'rope_variant'\u001b[0m: \u001b[32m'rope'\u001b[0m,\n",
            "    \u001b[32m'shortrope_length'\u001b[0m: \u001b[1;36m16\u001b[0m,\n",
            "    \u001b[32m'use_abs_pos_embeddings'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_attn'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_output'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_beta'\u001b[0m: \u001b[1;36m0.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_gamma'\u001b[0m: \u001b[1;36m100.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_use_euler_base'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'constantmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_x_intercept'\u001b[0m: \u001b[1;36m-100.0\u001b[0m,\n",
            "    \u001b[32m'polymax_y_intercept'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n",
            "    \u001b[32m'polymax_power'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_divisor'\u001b[0m: \u001b[1;36m1000.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_use_euler_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'strongermax_strength'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'softermax_use_xmax'\u001b[0m: \u001b[3;92mTrue\u001b[0m\n",
            "\u001b[1m}\u001b[0m\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "number of parameters: 10.65M\n",
            "num decayed parameter tensors: 26, with 10,740,096 parameters\n",
            "num non-decayed parameter tensors: 13, with 4,992 parameters\n",
            "using fused AdamW: True\n",
            "step \u001b[1;36m0\u001b[0m: train loss \u001b[1;36m4.2965\u001b[0m, val loss \u001b[1;36m4.2988\u001b[0m\n",
            "iter \u001b[1;36m0\u001b[0m: loss \u001b[1;36m4.2670\u001b[0m, time \u001b[1;36m17903.67\u001b[0m ms, mfu \u001b[1;36m-100.00\u001b[0m%\n",
            "iter \u001b[1;36m10\u001b[0m: loss \u001b[1;36m3.0281\u001b[0m, time \u001b[1;36m154.28\u001b[0m ms, mfu \u001b[1;36m2.42\u001b[0m%\n",
            "iter \u001b[1;36m20\u001b[0m: loss \u001b[1;36m2.6496\u001b[0m, time \u001b[1;36m156.51\u001b[0m ms, mfu \u001b[1;36m2.41\u001b[0m%\n",
            "iter \u001b[1;36m30\u001b[0m: loss \u001b[1;36m2.5883\u001b[0m, time \u001b[1;36m157.04\u001b[0m ms, mfu \u001b[1;36m2.41\u001b[0m%\n",
            "iter \u001b[1;36m40\u001b[0m: loss \u001b[1;36m2.5595\u001b[0m, time \u001b[1;36m156.61\u001b[0m ms, mfu \u001b[1;36m2.41\u001b[0m%\n",
            "iter \u001b[1;36m50\u001b[0m: loss \u001b[1;36m2.5243\u001b[0m, time \u001b[1;36m156.74\u001b[0m ms, mfu \u001b[1;36m2.40\u001b[0m%\n",
            "iter \u001b[1;36m60\u001b[0m: loss \u001b[1;36m2.5056\u001b[0m, time \u001b[1;36m154.15\u001b[0m ms, mfu \u001b[1;36m2.40\u001b[0m%\n",
            "iter \u001b[1;36m70\u001b[0m: loss \u001b[1;36m2.4938\u001b[0m, time \u001b[1;36m159.68\u001b[0m ms, mfu \u001b[1;36m2.40\u001b[0m%\n",
            "iter \u001b[1;36m80\u001b[0m: loss \u001b[1;36m2.5001\u001b[0m, time \u001b[1;36m156.58\u001b[0m ms, mfu \u001b[1;36m2.40\u001b[0m%\n",
            "iter \u001b[1;36m90\u001b[0m: loss \u001b[1;36m2.4723\u001b[0m, time \u001b[1;36m153.78\u001b[0m ms, mfu \u001b[1;36m2.40\u001b[0m%\n",
            "iter \u001b[1;36m100\u001b[0m: loss \u001b[1;36m2.4689\u001b[0m, time \u001b[1;36m155.62\u001b[0m ms, mfu \u001b[1;36m2.40\u001b[0m%\n",
            "iter \u001b[1;36m110\u001b[0m: loss \u001b[1;36m2.4616\u001b[0m, time \u001b[1;36m159.48\u001b[0m ms, mfu \u001b[1;36m2.39\u001b[0m%\n",
            "iter \u001b[1;36m120\u001b[0m: loss \u001b[1;36m2.4513\u001b[0m, time \u001b[1;36m159.17\u001b[0m ms, mfu \u001b[1;36m2.39\u001b[0m%\n",
            "iter \u001b[1;36m130\u001b[0m: loss \u001b[1;36m2.4423\u001b[0m, time \u001b[1;36m157.85\u001b[0m ms, mfu \u001b[1;36m2.38\u001b[0m%\n",
            "iter \u001b[1;36m140\u001b[0m: loss \u001b[1;36m2.4456\u001b[0m, time \u001b[1;36m159.09\u001b[0m ms, mfu \u001b[1;36m2.38\u001b[0m%\n",
            "iter \u001b[1;36m150\u001b[0m: loss \u001b[1;36m2.4605\u001b[0m, time \u001b[1;36m156.42\u001b[0m ms, mfu \u001b[1;36m2.38\u001b[0m%\n",
            "iter \u001b[1;36m160\u001b[0m: loss \u001b[1;36m2.4473\u001b[0m, time \u001b[1;36m153.20\u001b[0m ms, mfu \u001b[1;36m2.39\u001b[0m%\n",
            "iter \u001b[1;36m170\u001b[0m: loss \u001b[1;36m2.4282\u001b[0m, time \u001b[1;36m154.97\u001b[0m ms, mfu \u001b[1;36m2.39\u001b[0m%\n",
            "iter \u001b[1;36m180\u001b[0m: loss \u001b[1;36m2.4372\u001b[0m, time \u001b[1;36m159.25\u001b[0m ms, mfu \u001b[1;36m2.38\u001b[0m%\n",
            "iter \u001b[1;36m190\u001b[0m: loss \u001b[1;36m2.4133\u001b[0m, time \u001b[1;36m156.85\u001b[0m ms, mfu \u001b[1;36m2.38\u001b[0m%\n",
            "iter \u001b[1;36m200\u001b[0m: loss \u001b[1;36m2.4027\u001b[0m, time \u001b[1;36m158.04\u001b[0m ms, mfu \u001b[1;36m2.38\u001b[0m%\n",
            "iter \u001b[1;36m210\u001b[0m: loss \u001b[1;36m2.3686\u001b[0m, time \u001b[1;36m157.51\u001b[0m ms, mfu \u001b[1;36m2.38\u001b[0m%\n",
            "iter \u001b[1;36m220\u001b[0m: loss \u001b[1;36m2.4063\u001b[0m, time \u001b[1;36m159.63\u001b[0m ms, mfu \u001b[1;36m2.37\u001b[0m%\n",
            "iter \u001b[1;36m230\u001b[0m: loss \u001b[1;36m2.3840\u001b[0m, time \u001b[1;36m157.51\u001b[0m ms, mfu \u001b[1;36m2.37\u001b[0m%\n",
            "iter \u001b[1;36m240\u001b[0m: loss \u001b[1;36m2.4094\u001b[0m, time \u001b[1;36m159.64\u001b[0m ms, mfu \u001b[1;36m2.37\u001b[0m%\n",
            "step \u001b[1;36m250\u001b[0m: train loss \u001b[1;36m2.3300\u001b[0m, val loss \u001b[1;36m2.3567\u001b[0m\n",
            "saving checkpoint to out-shakespeare\n",
            "iter \u001b[1;36m250\u001b[0m: loss \u001b[1;36m2.3620\u001b[0m, time \u001b[1;36m16747.51\u001b[0m ms, mfu \u001b[1;36m2.13\u001b[0m%\n",
            "iter \u001b[1;36m260\u001b[0m: loss \u001b[1;36m2.3488\u001b[0m, time \u001b[1;36m160.14\u001b[0m ms, mfu \u001b[1;36m2.15\u001b[0m%\n",
            "iter \u001b[1;36m270\u001b[0m: loss \u001b[1;36m2.3482\u001b[0m, time \u001b[1;36m162.37\u001b[0m ms, mfu \u001b[1;36m2.17\u001b[0m%\n",
            "iter \u001b[1;36m280\u001b[0m: loss \u001b[1;36m2.3560\u001b[0m, time \u001b[1;36m163.51\u001b[0m ms, mfu \u001b[1;36m2.18\u001b[0m%\n",
            "iter \u001b[1;36m290\u001b[0m: loss \u001b[1;36m2.3114\u001b[0m, time \u001b[1;36m161.49\u001b[0m ms, mfu \u001b[1;36m2.19\u001b[0m%\n",
            "iter \u001b[1;36m300\u001b[0m: loss \u001b[1;36m2.3034\u001b[0m, time \u001b[1;36m161.24\u001b[0m ms, mfu \u001b[1;36m2.20\u001b[0m%\n",
            "iter \u001b[1;36m310\u001b[0m: loss \u001b[1;36m2.2813\u001b[0m, time \u001b[1;36m163.18\u001b[0m ms, mfu \u001b[1;36m2.21\u001b[0m%\n",
            "iter \u001b[1;36m320\u001b[0m: loss \u001b[1;36m2.2898\u001b[0m, time \u001b[1;36m162.72\u001b[0m ms, mfu \u001b[1;36m2.22\u001b[0m%\n",
            "iter \u001b[1;36m330\u001b[0m: loss \u001b[1;36m2.2425\u001b[0m, time \u001b[1;36m163.96\u001b[0m ms, mfu \u001b[1;36m2.22\u001b[0m%\n",
            "iter \u001b[1;36m340\u001b[0m: loss \u001b[1;36m2.2145\u001b[0m, time \u001b[1;36m162.96\u001b[0m ms, mfu \u001b[1;36m2.23\u001b[0m%\n",
            "iter \u001b[1;36m350\u001b[0m: loss \u001b[1;36m2.2455\u001b[0m, time \u001b[1;36m162.72\u001b[0m ms, mfu \u001b[1;36m2.24\u001b[0m%\n",
            "iter \u001b[1;36m360\u001b[0m: loss \u001b[1;36m2.1902\u001b[0m, time \u001b[1;36m161.86\u001b[0m ms, mfu \u001b[1;36m2.24\u001b[0m%\n",
            "iter \u001b[1;36m370\u001b[0m: loss \u001b[1;36m2.1803\u001b[0m, time \u001b[1;36m163.79\u001b[0m ms, mfu \u001b[1;36m2.25\u001b[0m%\n",
            "iter \u001b[1;36m380\u001b[0m: loss \u001b[1;36m2.1495\u001b[0m, time \u001b[1;36m164.31\u001b[0m ms, mfu \u001b[1;36m2.25\u001b[0m%\n",
            "iter \u001b[1;36m390\u001b[0m: loss \u001b[1;36m2.1668\u001b[0m, time \u001b[1;36m166.52\u001b[0m ms, mfu \u001b[1;36m2.25\u001b[0m%\n",
            "iter \u001b[1;36m400\u001b[0m: loss \u001b[1;36m2.1530\u001b[0m, time \u001b[1;36m166.19\u001b[0m ms, mfu \u001b[1;36m2.25\u001b[0m%\n",
            "iter \u001b[1;36m410\u001b[0m: loss \u001b[1;36m2.1149\u001b[0m, time \u001b[1;36m165.96\u001b[0m ms, mfu \u001b[1;36m2.25\u001b[0m%\n",
            "iter \u001b[1;36m420\u001b[0m: loss \u001b[1;36m2.1272\u001b[0m, time \u001b[1;36m167.59\u001b[0m ms, mfu \u001b[1;36m2.24\u001b[0m%\n",
            "iter \u001b[1;36m430\u001b[0m: loss \u001b[1;36m2.1005\u001b[0m, time \u001b[1;36m165.42\u001b[0m ms, mfu \u001b[1;36m2.25\u001b[0m%\n",
            "iter \u001b[1;36m440\u001b[0m: loss \u001b[1;36m2.0515\u001b[0m, time \u001b[1;36m169.25\u001b[0m ms, mfu \u001b[1;36m2.24\u001b[0m%\n",
            "iter \u001b[1;36m450\u001b[0m: loss \u001b[1;36m2.0579\u001b[0m, time \u001b[1;36m168.64\u001b[0m ms, mfu \u001b[1;36m2.24\u001b[0m%\n",
            "iter \u001b[1;36m460\u001b[0m: loss \u001b[1;36m2.0059\u001b[0m, time \u001b[1;36m173.24\u001b[0m ms, mfu \u001b[1;36m2.23\u001b[0m%\n",
            "iter \u001b[1;36m470\u001b[0m: loss \u001b[1;36m2.0431\u001b[0m, time \u001b[1;36m169.52\u001b[0m ms, mfu \u001b[1;36m2.23\u001b[0m%\n",
            "iter \u001b[1;36m480\u001b[0m: loss \u001b[1;36m1.9882\u001b[0m, time \u001b[1;36m171.02\u001b[0m ms, mfu \u001b[1;36m2.22\u001b[0m%\n",
            "iter \u001b[1;36m490\u001b[0m: loss \u001b[1;36m2.0564\u001b[0m, time \u001b[1;36m172.65\u001b[0m ms, mfu \u001b[1;36m2.21\u001b[0m%\n",
            "step \u001b[1;36m500\u001b[0m: train loss \u001b[1;36m1.9398\u001b[0m, val loss \u001b[1;36m2.0429\u001b[0m\n",
            "saving checkpoint to out-shakespeare\n",
            "iter \u001b[1;36m500\u001b[0m: loss \u001b[1;36m2.0108\u001b[0m, time \u001b[1;36m19292.24\u001b[0m ms, mfu \u001b[1;36m2.00\u001b[0m%\n",
            "iter \u001b[1;36m510\u001b[0m: loss \u001b[1;36m1.9870\u001b[0m, time \u001b[1;36m172.71\u001b[0m ms, mfu \u001b[1;36m2.01\u001b[0m%\n",
            "iter \u001b[1;36m520\u001b[0m: loss \u001b[1;36m1.9619\u001b[0m, time \u001b[1;36m172.48\u001b[0m ms, mfu \u001b[1;36m2.03\u001b[0m%\n",
            "iter \u001b[1;36m530\u001b[0m: loss \u001b[1;36m1.9379\u001b[0m, time \u001b[1;36m174.54\u001b[0m ms, mfu \u001b[1;36m2.04\u001b[0m%\n",
            "iter \u001b[1;36m540\u001b[0m: loss \u001b[1;36m1.9649\u001b[0m, time \u001b[1;36m174.22\u001b[0m ms, mfu \u001b[1;36m2.05\u001b[0m%\n",
            "iter \u001b[1;36m550\u001b[0m: loss \u001b[1;36m1.9100\u001b[0m, time \u001b[1;36m171.34\u001b[0m ms, mfu \u001b[1;36m2.06\u001b[0m%\n",
            "iter \u001b[1;36m560\u001b[0m: loss \u001b[1;36m1.9115\u001b[0m, time \u001b[1;36m169.21\u001b[0m ms, mfu \u001b[1;36m2.07\u001b[0m%\n",
            "iter \u001b[1;36m570\u001b[0m: loss \u001b[1;36m1.9097\u001b[0m, time \u001b[1;36m171.39\u001b[0m ms, mfu \u001b[1;36m2.08\u001b[0m%\n",
            "iter \u001b[1;36m580\u001b[0m: loss \u001b[1;36m1.8975\u001b[0m, time \u001b[1;36m169.48\u001b[0m ms, mfu \u001b[1;36m2.10\u001b[0m%\n",
            "iter \u001b[1;36m590\u001b[0m: loss \u001b[1;36m1.8533\u001b[0m, time \u001b[1;36m172.77\u001b[0m ms, mfu \u001b[1;36m2.10\u001b[0m%\n",
            "iter \u001b[1;36m600\u001b[0m: loss \u001b[1;36m1.8624\u001b[0m, time \u001b[1;36m172.23\u001b[0m ms, mfu \u001b[1;36m2.11\u001b[0m%\n",
            "iter \u001b[1;36m610\u001b[0m: loss \u001b[1;36m1.8804\u001b[0m, time \u001b[1;36m170.44\u001b[0m ms, mfu \u001b[1;36m2.12\u001b[0m%\n",
            "iter \u001b[1;36m620\u001b[0m: loss \u001b[1;36m1.8627\u001b[0m, time \u001b[1;36m169.69\u001b[0m ms, mfu \u001b[1;36m2.12\u001b[0m%\n",
            "iter \u001b[1;36m630\u001b[0m: loss \u001b[1;36m1.8562\u001b[0m, time \u001b[1;36m170.97\u001b[0m ms, mfu \u001b[1;36m2.13\u001b[0m%\n",
            "iter \u001b[1;36m640\u001b[0m: loss \u001b[1;36m1.8030\u001b[0m, time \u001b[1;36m172.57\u001b[0m ms, mfu \u001b[1;36m2.13\u001b[0m%\n",
            "iter \u001b[1;36m650\u001b[0m: loss \u001b[1;36m1.8408\u001b[0m, time \u001b[1;36m171.68\u001b[0m ms, mfu \u001b[1;36m2.14\u001b[0m%\n",
            "iter \u001b[1;36m660\u001b[0m: loss \u001b[1;36m1.8318\u001b[0m, time \u001b[1;36m174.55\u001b[0m ms, mfu \u001b[1;36m2.14\u001b[0m%\n",
            "iter \u001b[1;36m670\u001b[0m: loss \u001b[1;36m1.7779\u001b[0m, time \u001b[1;36m175.46\u001b[0m ms, mfu \u001b[1;36m2.13\u001b[0m%\n",
            "iter \u001b[1;36m680\u001b[0m: loss \u001b[1;36m1.8308\u001b[0m, time \u001b[1;36m173.06\u001b[0m ms, mfu \u001b[1;36m2.14\u001b[0m%\n",
            "iter \u001b[1;36m690\u001b[0m: loss \u001b[1;36m1.7816\u001b[0m, time \u001b[1;36m168.74\u001b[0m ms, mfu \u001b[1;36m2.14\u001b[0m%\n",
            "iter \u001b[1;36m700\u001b[0m: loss \u001b[1;36m1.8090\u001b[0m, time \u001b[1;36m174.73\u001b[0m ms, mfu \u001b[1;36m2.14\u001b[0m%\n",
            "iter \u001b[1;36m710\u001b[0m: loss \u001b[1;36m1.7757\u001b[0m, time \u001b[1;36m173.97\u001b[0m ms, mfu \u001b[1;36m2.14\u001b[0m%\n",
            "iter \u001b[1;36m720\u001b[0m: loss \u001b[1;36m1.7544\u001b[0m, time \u001b[1;36m175.82\u001b[0m ms, mfu \u001b[1;36m2.14\u001b[0m%\n",
            "iter \u001b[1;36m730\u001b[0m: loss \u001b[1;36m1.7404\u001b[0m, time \u001b[1;36m174.70\u001b[0m ms, mfu \u001b[1;36m2.14\u001b[0m%\n",
            "iter \u001b[1;36m740\u001b[0m: loss \u001b[1;36m1.7317\u001b[0m, time \u001b[1;36m176.66\u001b[0m ms, mfu \u001b[1;36m2.14\u001b[0m%\n",
            "step \u001b[1;36m750\u001b[0m: train loss \u001b[1;36m1.6719\u001b[0m, val loss \u001b[1;36m1.8514\u001b[0m\n",
            "saving checkpoint to out-shakespeare\n",
            "iter \u001b[1;36m750\u001b[0m: loss \u001b[1;36m1.7325\u001b[0m, time \u001b[1;36m18982.01\u001b[0m ms, mfu \u001b[1;36m1.92\u001b[0m%\n",
            "iter \u001b[1;36m760\u001b[0m: loss \u001b[1;36m1.7396\u001b[0m, time \u001b[1;36m171.64\u001b[0m ms, mfu \u001b[1;36m1.95\u001b[0m%\n",
            "iter \u001b[1;36m770\u001b[0m: loss \u001b[1;36m1.7474\u001b[0m, time \u001b[1;36m174.98\u001b[0m ms, mfu \u001b[1;36m1.97\u001b[0m%\n",
            "iter \u001b[1;36m780\u001b[0m: loss \u001b[1;36m1.7243\u001b[0m, time \u001b[1;36m173.05\u001b[0m ms, mfu \u001b[1;36m1.99\u001b[0m%\n",
            "iter \u001b[1;36m790\u001b[0m: loss \u001b[1;36m1.7193\u001b[0m, time \u001b[1;36m177.03\u001b[0m ms, mfu \u001b[1;36m2.00\u001b[0m%\n",
            "iter \u001b[1;36m800\u001b[0m: loss \u001b[1;36m1.7176\u001b[0m, time \u001b[1;36m171.38\u001b[0m ms, mfu \u001b[1;36m2.02\u001b[0m%\n",
            "iter \u001b[1;36m810\u001b[0m: loss \u001b[1;36m1.6988\u001b[0m, time \u001b[1;36m176.67\u001b[0m ms, mfu \u001b[1;36m2.02\u001b[0m%\n",
            "iter \u001b[1;36m820\u001b[0m: loss \u001b[1;36m1.6975\u001b[0m, time \u001b[1;36m174.67\u001b[0m ms, mfu \u001b[1;36m2.04\u001b[0m%\n",
            "iter \u001b[1;36m830\u001b[0m: loss \u001b[1;36m1.6967\u001b[0m, time \u001b[1;36m173.96\u001b[0m ms, mfu \u001b[1;36m2.05\u001b[0m%\n",
            "iter \u001b[1;36m840\u001b[0m: loss \u001b[1;36m1.6964\u001b[0m, time \u001b[1;36m177.38\u001b[0m ms, mfu \u001b[1;36m2.05\u001b[0m%\n",
            "iter \u001b[1;36m850\u001b[0m: loss \u001b[1;36m1.6771\u001b[0m, time \u001b[1;36m173.83\u001b[0m ms, mfu \u001b[1;36m2.06\u001b[0m%\n",
            "iter \u001b[1;36m860\u001b[0m: loss \u001b[1;36m1.6841\u001b[0m, time \u001b[1;36m170.01\u001b[0m ms, mfu \u001b[1;36m2.07\u001b[0m%\n",
            "iter \u001b[1;36m870\u001b[0m: loss \u001b[1;36m1.6843\u001b[0m, time \u001b[1;36m173.03\u001b[0m ms, mfu \u001b[1;36m2.08\u001b[0m%\n",
            "iter \u001b[1;36m880\u001b[0m: loss \u001b[1;36m1.6747\u001b[0m, time \u001b[1;36m173.04\u001b[0m ms, mfu \u001b[1;36m2.09\u001b[0m%\n",
            "iter \u001b[1;36m890\u001b[0m: loss \u001b[1;36m1.6814\u001b[0m, time \u001b[1;36m173.87\u001b[0m ms, mfu \u001b[1;36m2.09\u001b[0m%\n",
            "iter \u001b[1;36m900\u001b[0m: loss \u001b[1;36m1.6749\u001b[0m, time \u001b[1;36m174.36\u001b[0m ms, mfu \u001b[1;36m2.10\u001b[0m%\n",
            "iter \u001b[1;36m910\u001b[0m: loss \u001b[1;36m1.6103\u001b[0m, time \u001b[1;36m173.83\u001b[0m ms, mfu \u001b[1;36m2.10\u001b[0m%\n",
            "iter \u001b[1;36m920\u001b[0m: loss \u001b[1;36m1.6526\u001b[0m, time \u001b[1;36m173.71\u001b[0m ms, mfu \u001b[1;36m2.11\u001b[0m%\n",
            "iter \u001b[1;36m930\u001b[0m: loss \u001b[1;36m1.6435\u001b[0m, time \u001b[1;36m176.58\u001b[0m ms, mfu \u001b[1;36m2.11\u001b[0m%\n",
            "iter \u001b[1;36m940\u001b[0m: loss \u001b[1;36m1.6292\u001b[0m, time \u001b[1;36m173.09\u001b[0m ms, mfu \u001b[1;36m2.11\u001b[0m%\n",
            "iter \u001b[1;36m950\u001b[0m: loss \u001b[1;36m1.6380\u001b[0m, time \u001b[1;36m177.30\u001b[0m ms, mfu \u001b[1;36m2.11\u001b[0m%\n",
            "iter \u001b[1;36m960\u001b[0m: loss \u001b[1;36m1.6622\u001b[0m, time \u001b[1;36m175.28\u001b[0m ms, mfu \u001b[1;36m2.11\u001b[0m%\n",
            "iter \u001b[1;36m970\u001b[0m: loss \u001b[1;36m1.6559\u001b[0m, time \u001b[1;36m174.02\u001b[0m ms, mfu \u001b[1;36m2.12\u001b[0m%\n",
            "iter \u001b[1;36m980\u001b[0m: loss \u001b[1;36m1.6372\u001b[0m, time \u001b[1;36m175.44\u001b[0m ms, mfu \u001b[1;36m2.12\u001b[0m%\n",
            "iter \u001b[1;36m990\u001b[0m: loss \u001b[1;36m1.6234\u001b[0m, time \u001b[1;36m174.55\u001b[0m ms, mfu \u001b[1;36m2.12\u001b[0m%\n",
            "step \u001b[1;36m1000\u001b[0m: train loss \u001b[1;36m1.5572\u001b[0m, val loss \u001b[1;36m1.7457\u001b[0m\n",
            "saving checkpoint to out-shakespeare\n",
            "iter \u001b[1;36m1000\u001b[0m: loss \u001b[1;36m1.6146\u001b[0m, time \u001b[1;36m19029.56\u001b[0m ms, mfu \u001b[1;36m1.91\u001b[0m%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FsR_dPeVyVJ8"
      },
      "source": [
        "<div class=\"mrkdown-google-sans\">\n",
        "\n",
        "### **Run GPU Inference**\n",
        "\n",
        "</div>\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python3 sample.py --out_dir=\"out-shakespeare\" --device=\"cuda\" --dtype=\"float16\" --num_samples=1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UkqxTfU_yUrK",
        "outputId": "b85133e0-7844-4832-9d81-c6465170ee31"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1m{\u001b[0m\n",
            "    \u001b[32m'block_size'\u001b[0m: \u001b[1;36m256\u001b[0m,\n",
            "    \u001b[32m'n_layer'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_head'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_embd'\u001b[0m: \u001b[1;36m384\u001b[0m,\n",
            "    \u001b[32m'dropout'\u001b[0m: \u001b[1;36m0.0\u001b[0m,\n",
            "    \u001b[32m'use_post_ln'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'layernorm_variant'\u001b[0m: \u001b[32m'rmsnorm'\u001b[0m,\n",
            "    \u001b[32m'bias'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'activation_variant'\u001b[0m: \u001b[32m'gelu'\u001b[0m,\n",
            "    \u001b[32m'use_rotary_embeddings'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'rope_variant'\u001b[0m: \u001b[32m'rope'\u001b[0m,\n",
            "    \u001b[32m'shortrope_length'\u001b[0m: \u001b[1;36m16\u001b[0m,\n",
            "    \u001b[32m'use_abs_pos_embeddings'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_attn'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_output'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_beta'\u001b[0m: \u001b[1;36m0.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_gamma'\u001b[0m: \u001b[1;36m100.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_use_euler_base'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'constantmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_x_intercept'\u001b[0m: \u001b[1;36m-100.0\u001b[0m,\n",
            "    \u001b[32m'polymax_y_intercept'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n",
            "    \u001b[32m'polymax_power'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_divisor'\u001b[0m: \u001b[1;36m1000.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_use_euler_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'strongermax_strength'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'softermax_use_xmax'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'vocab_size'\u001b[0m: \u001b[1;36m65\u001b[0m\n",
            "\u001b[1m}\u001b[0m\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "number of parameters: 10.65M\n",
            "Loading meta from data/shakespeare_char/meta.pkl\u001b[33m...\u001b[0m\n",
            "\n",
            "\u001b[1;32mAnd the Romeo own, which a be madest,\u001b[0m\n",
            "\u001b[1;32mAnd but take rud my day taught their foul him to beceding\u001b[0m\n",
            "\u001b[1;32mHath away, my facestation. The now, from that I comman;\u001b[0m\n",
            "\u001b[1;32mI lord their enten command Here to by bear be none,\u001b[0m\n",
            "\u001b[1;32mWill and self in you hith she coursey:\u001b[0m\n",
            "\u001b[1;32mAs that speak you love us now pate at sweep\u001b[0m\n",
            "\u001b[1;32mIs would that\u001b[0m\n",
            "\u001b[1;32mTo Will the withinks the madestive with in his so more for unto and thrust for the\u001b[0m\n",
            "\u001b[1;32mgreath: my fear leave, that Prive my of the golder still is:\u001b[0m\n",
            "\u001b[1;32mArd all be Edward him be traithere of you\u001b[0m\n",
            "\u001b[1;32mmoster for th\u001b[0m\n",
            "---------------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<div class=\"mrkdown-google-sans\">\n",
        "\n",
        "# **Run Exploration**\n",
        "\n",
        "</div>"
      ],
      "metadata": {
        "id": "oGjK3xvh09AP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%cd\n",
        "%cd nanoGPT_gpu\n",
        "\n",
        "!cat explorations/config.json"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wUdV_T2V1Z6U",
        "outputId": "321288c1-f6c4-4fa3-80dd-461c67dcf093"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/root\n",
            "/root/nanoGPT_gpu\n",
            "[\n",
            "  {\n",
            "    \"max_iters\": [\"500\"],\n",
            "    \"n_layer\": [\"6\"],\n",
            "    \"n_head\": [\"6\"],\n",
            "    \"n_embd\": [\"384\"],\n",
            "    \"block_size\":[\"64\",\"128\",\"256\",\"512\"],\n",
            "    \"device\": [\"cuda\"],\n",
            "    \"dtype\": [\"float16\"],\n",
            "    \"dataset\": [\"shakespeare_char\"],\n",
            "    \"use_rotary_embeddings\": [false],\n",
            "    \"use_abs_pos_embeddings\": [true],\n",
            "    \"tensorboard_run_name\": \"shakespeare_context_length_sweep\"\n",
            "  }\n",
            "]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python3 run_experiments.py --config explorations/config.json --value_only --output_dir out_test"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cb0444ff-3430-4472-d6c8-1ded75c3f29d",
        "id": "Ujt8vtiaLkS9"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Running command: python3 train.py --max_iters 500 --n_layer 6 --n_head 6 --n_embd 384 --block_size 64 --device cuda --dtype float16 --dataset shakespeare_char --no-use_rotary_embeddings --use_abs_pos_embeddings --tensorboard_run_name config-500-6-6-384-64-cuda-float16-shakespeare_char-False-True-s --out_dir out_test/20240120_002732_config-500-6-6-384-64-cuda-float16-shakespeare_char-False-True-s\n",
            "2024-01-20 00:27:34.315624: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-01-20 00:27:34.315679: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-01-20 00:27:34.316937: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-01-20 00:27:34.323935: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-01-20 00:27:35.328316: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "\u001b[1m{\u001b[0m\n",
            "    \u001b[32m'block_size'\u001b[0m: \u001b[1;36m64\u001b[0m,\n",
            "    \u001b[32m'n_layer'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_head'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_embd'\u001b[0m: \u001b[1;36m384\u001b[0m,\n",
            "    \u001b[32m'dropout'\u001b[0m: \u001b[1;36m0.2\u001b[0m,\n",
            "    \u001b[32m'use_post_ln'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'layernorm_variant'\u001b[0m: \u001b[32m'rmsnorm'\u001b[0m,\n",
            "    \u001b[32m'bias'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'activation_variant'\u001b[0m: \u001b[32m'gelu'\u001b[0m,\n",
            "    \u001b[32m'use_rotary_embeddings'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'rope_variant'\u001b[0m: \u001b[32m'rope'\u001b[0m,\n",
            "    \u001b[32m'shortrope_length'\u001b[0m: \u001b[1;36m16\u001b[0m,\n",
            "    \u001b[32m'use_abs_pos_embeddings'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_attn'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_output'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_beta'\u001b[0m: \u001b[1;36m0.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_gamma'\u001b[0m: \u001b[1;36m100.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_use_euler_base'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'constantmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_x_intercept'\u001b[0m: \u001b[1;36m-100.0\u001b[0m,\n",
            "    \u001b[32m'polymax_y_intercept'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n",
            "    \u001b[32m'polymax_power'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_divisor'\u001b[0m: \u001b[1;36m1000.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_use_euler_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'strongermax_strength'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'softermax_use_xmax'\u001b[0m: \u001b[3;92mTrue\u001b[0m\n",
            "\u001b[1m}\u001b[0m\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "number of parameters: 10.65M\n",
            "num decayed parameter tensors: 26, with 10,666,368 parameters\n",
            "num non-decayed parameter tensors: 13, with 4,992 parameters\n",
            "using fused AdamW: True\n",
            "step \u001b[1;36m0\u001b[0m: train loss \u001b[1;36m4.3242\u001b[0m, val loss \u001b[1;36m4.3190\u001b[0m\n",
            "iter \u001b[1;36m0\u001b[0m: loss \u001b[1;36m4.3068\u001b[0m, time \u001b[1;36m5344.74\u001b[0m ms, mfu \u001b[1;36m-100.00\u001b[0m%\n",
            "iter \u001b[1;36m10\u001b[0m: loss \u001b[1;36m3.3844\u001b[0m, time \u001b[1;36m47.33\u001b[0m ms, mfu \u001b[1;36m1.82\u001b[0m%\n",
            "iter \u001b[1;36m20\u001b[0m: loss \u001b[1;36m3.1978\u001b[0m, time \u001b[1;36m46.75\u001b[0m ms, mfu \u001b[1;36m1.82\u001b[0m%\n",
            "iter \u001b[1;36m30\u001b[0m: loss \u001b[1;36m2.9954\u001b[0m, time \u001b[1;36m47.60\u001b[0m ms, mfu \u001b[1;36m1.82\u001b[0m%\n",
            "iter \u001b[1;36m40\u001b[0m: loss \u001b[1;36m2.8807\u001b[0m, time \u001b[1;36m48.16\u001b[0m ms, mfu \u001b[1;36m1.82\u001b[0m%\n",
            "iter \u001b[1;36m50\u001b[0m: loss \u001b[1;36m2.7464\u001b[0m, time \u001b[1;36m47.33\u001b[0m ms, mfu \u001b[1;36m1.82\u001b[0m%\n",
            "iter \u001b[1;36m60\u001b[0m: loss \u001b[1;36m2.6839\u001b[0m, time \u001b[1;36m48.22\u001b[0m ms, mfu \u001b[1;36m1.82\u001b[0m%\n",
            "iter \u001b[1;36m70\u001b[0m: loss \u001b[1;36m2.6746\u001b[0m, time \u001b[1;36m46.61\u001b[0m ms, mfu \u001b[1;36m1.82\u001b[0m%\n",
            "iter \u001b[1;36m80\u001b[0m: loss \u001b[1;36m2.6093\u001b[0m, time \u001b[1;36m46.82\u001b[0m ms, mfu \u001b[1;36m1.82\u001b[0m%\n",
            "iter \u001b[1;36m90\u001b[0m: loss \u001b[1;36m2.5877\u001b[0m, time \u001b[1;36m47.37\u001b[0m ms, mfu \u001b[1;36m1.82\u001b[0m%\n",
            "iter \u001b[1;36m100\u001b[0m: loss \u001b[1;36m2.5484\u001b[0m, time \u001b[1;36m45.50\u001b[0m ms, mfu \u001b[1;36m1.83\u001b[0m%\n",
            "iter \u001b[1;36m110\u001b[0m: loss \u001b[1;36m2.5027\u001b[0m, time \u001b[1;36m47.05\u001b[0m ms, mfu \u001b[1;36m1.83\u001b[0m%\n",
            "iter \u001b[1;36m120\u001b[0m: loss \u001b[1;36m2.5362\u001b[0m, time \u001b[1;36m46.87\u001b[0m ms, mfu \u001b[1;36m1.83\u001b[0m%\n",
            "iter \u001b[1;36m130\u001b[0m: loss \u001b[1;36m2.4874\u001b[0m, time \u001b[1;36m46.54\u001b[0m ms, mfu \u001b[1;36m1.83\u001b[0m%\n",
            "iter \u001b[1;36m140\u001b[0m: loss \u001b[1;36m2.4893\u001b[0m, time \u001b[1;36m47.23\u001b[0m ms, mfu \u001b[1;36m1.83\u001b[0m%\n",
            "iter \u001b[1;36m150\u001b[0m: loss \u001b[1;36m2.4143\u001b[0m, time \u001b[1;36m47.93\u001b[0m ms, mfu \u001b[1;36m1.83\u001b[0m%\n",
            "iter \u001b[1;36m160\u001b[0m: loss \u001b[1;36m2.4283\u001b[0m, time \u001b[1;36m46.84\u001b[0m ms, mfu \u001b[1;36m1.83\u001b[0m%\n",
            "iter \u001b[1;36m170\u001b[0m: loss \u001b[1;36m2.4257\u001b[0m, time \u001b[1;36m48.77\u001b[0m ms, mfu \u001b[1;36m1.82\u001b[0m%\n",
            "iter \u001b[1;36m180\u001b[0m: loss \u001b[1;36m2.3791\u001b[0m, time \u001b[1;36m45.95\u001b[0m ms, mfu \u001b[1;36m1.83\u001b[0m%\n",
            "iter \u001b[1;36m190\u001b[0m: loss \u001b[1;36m2.3323\u001b[0m, time \u001b[1;36m46.92\u001b[0m ms, mfu \u001b[1;36m1.83\u001b[0m%\n",
            "iter \u001b[1;36m200\u001b[0m: loss \u001b[1;36m2.3424\u001b[0m, time \u001b[1;36m47.33\u001b[0m ms, mfu \u001b[1;36m1.83\u001b[0m%\n",
            "iter \u001b[1;36m210\u001b[0m: loss \u001b[1;36m2.3257\u001b[0m, time \u001b[1;36m48.71\u001b[0m ms, mfu \u001b[1;36m1.82\u001b[0m%\n",
            "iter \u001b[1;36m220\u001b[0m: loss \u001b[1;36m2.3630\u001b[0m, time \u001b[1;36m46.77\u001b[0m ms, mfu \u001b[1;36m1.82\u001b[0m%\n",
            "iter \u001b[1;36m230\u001b[0m: loss \u001b[1;36m2.3269\u001b[0m, time \u001b[1;36m48.33\u001b[0m ms, mfu \u001b[1;36m1.82\u001b[0m%\n",
            "iter \u001b[1;36m240\u001b[0m: loss \u001b[1;36m2.2751\u001b[0m, time \u001b[1;36m47.96\u001b[0m ms, mfu \u001b[1;36m1.82\u001b[0m%\n",
            "step \u001b[1;36m250\u001b[0m: train loss \u001b[1;36m2.1997\u001b[0m, val loss \u001b[1;36m2.2277\u001b[0m\n",
            "saving checkpoint to \n",
            "out_test/20240120_002732_config-\u001b[1;36m500\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m384\u001b[0m-\u001b[1;36m64\u001b[0m-cuda-float16-shakespeare_char-\u001b[3;91mFalse\u001b[0m-\u001b[3;92mTrue\u001b[0m-s\n",
            "iter \u001b[1;36m250\u001b[0m: loss \u001b[1;36m2.2827\u001b[0m, time \u001b[1;36m4605.77\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "iter \u001b[1;36m260\u001b[0m: loss \u001b[1;36m2.2728\u001b[0m, time \u001b[1;36m47.66\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m270\u001b[0m: loss \u001b[1;36m2.2380\u001b[0m, time \u001b[1;36m48.46\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m280\u001b[0m: loss \u001b[1;36m2.2038\u001b[0m, time \u001b[1;36m47.99\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m290\u001b[0m: loss \u001b[1;36m2.2445\u001b[0m, time \u001b[1;36m48.74\u001b[0m ms, mfu \u001b[1;36m1.69\u001b[0m%\n",
            "iter \u001b[1;36m300\u001b[0m: loss \u001b[1;36m2.2033\u001b[0m, time \u001b[1;36m48.49\u001b[0m ms, mfu \u001b[1;36m1.70\u001b[0m%\n",
            "iter \u001b[1;36m310\u001b[0m: loss \u001b[1;36m2.2223\u001b[0m, time \u001b[1;36m46.63\u001b[0m ms, mfu \u001b[1;36m1.71\u001b[0m%\n",
            "iter \u001b[1;36m320\u001b[0m: loss \u001b[1;36m2.1843\u001b[0m, time \u001b[1;36m48.02\u001b[0m ms, mfu \u001b[1;36m1.72\u001b[0m%\n",
            "iter \u001b[1;36m330\u001b[0m: loss \u001b[1;36m2.1800\u001b[0m, time \u001b[1;36m47.92\u001b[0m ms, mfu \u001b[1;36m1.73\u001b[0m%\n",
            "iter \u001b[1;36m340\u001b[0m: loss \u001b[1;36m2.1070\u001b[0m, time \u001b[1;36m49.20\u001b[0m ms, mfu \u001b[1;36m1.73\u001b[0m%\n",
            "iter \u001b[1;36m350\u001b[0m: loss \u001b[1;36m2.1371\u001b[0m, time \u001b[1;36m48.87\u001b[0m ms, mfu \u001b[1;36m1.73\u001b[0m%\n",
            "iter \u001b[1;36m360\u001b[0m: loss \u001b[1;36m2.1435\u001b[0m, time \u001b[1;36m48.89\u001b[0m ms, mfu \u001b[1;36m1.74\u001b[0m%\n",
            "iter \u001b[1;36m370\u001b[0m: loss \u001b[1;36m2.1212\u001b[0m, time \u001b[1;36m47.70\u001b[0m ms, mfu \u001b[1;36m1.74\u001b[0m%\n",
            "iter \u001b[1;36m380\u001b[0m: loss \u001b[1;36m2.1330\u001b[0m, time \u001b[1;36m47.28\u001b[0m ms, mfu \u001b[1;36m1.75\u001b[0m%\n",
            "iter \u001b[1;36m390\u001b[0m: loss \u001b[1;36m2.0787\u001b[0m, time \u001b[1;36m48.32\u001b[0m ms, mfu \u001b[1;36m1.76\u001b[0m%\n",
            "iter \u001b[1;36m400\u001b[0m: loss \u001b[1;36m2.1050\u001b[0m, time \u001b[1;36m47.49\u001b[0m ms, mfu \u001b[1;36m1.76\u001b[0m%\n",
            "iter \u001b[1;36m410\u001b[0m: loss \u001b[1;36m2.0801\u001b[0m, time \u001b[1;36m47.55\u001b[0m ms, mfu \u001b[1;36m1.77\u001b[0m%\n",
            "iter \u001b[1;36m420\u001b[0m: loss \u001b[1;36m2.0883\u001b[0m, time \u001b[1;36m47.49\u001b[0m ms, mfu \u001b[1;36m1.77\u001b[0m%\n",
            "iter \u001b[1;36m430\u001b[0m: loss \u001b[1;36m2.0279\u001b[0m, time \u001b[1;36m47.62\u001b[0m ms, mfu \u001b[1;36m1.78\u001b[0m%\n",
            "iter \u001b[1;36m440\u001b[0m: loss \u001b[1;36m2.0148\u001b[0m, time \u001b[1;36m47.12\u001b[0m ms, mfu \u001b[1;36m1.78\u001b[0m%\n",
            "iter \u001b[1;36m450\u001b[0m: loss \u001b[1;36m2.0287\u001b[0m, time \u001b[1;36m49.80\u001b[0m ms, mfu \u001b[1;36m1.78\u001b[0m%\n",
            "iter \u001b[1;36m460\u001b[0m: loss \u001b[1;36m2.0280\u001b[0m, time \u001b[1;36m49.33\u001b[0m ms, mfu \u001b[1;36m1.77\u001b[0m%\n",
            "iter \u001b[1;36m470\u001b[0m: loss \u001b[1;36m1.9884\u001b[0m, time \u001b[1;36m49.57\u001b[0m ms, mfu \u001b[1;36m1.77\u001b[0m%\n",
            "iter \u001b[1;36m480\u001b[0m: loss \u001b[1;36m2.0140\u001b[0m, time \u001b[1;36m50.26\u001b[0m ms, mfu \u001b[1;36m1.76\u001b[0m%\n",
            "iter \u001b[1;36m490\u001b[0m: loss \u001b[1;36m2.0285\u001b[0m, time \u001b[1;36m49.16\u001b[0m ms, mfu \u001b[1;36m1.76\u001b[0m%\n",
            "step \u001b[1;36m500\u001b[0m: train loss \u001b[1;36m1.8897\u001b[0m, val loss \u001b[1;36m1.9912\u001b[0m\n",
            "saving checkpoint to \n",
            "out_test/20240120_002732_config-\u001b[1;36m500\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m384\u001b[0m-\u001b[1;36m64\u001b[0m-cuda-float16-shakespeare_char-\u001b[3;91mFalse\u001b[0m-\u001b[3;92mTrue\u001b[0m-s\n",
            "iter \u001b[1;36m500\u001b[0m: loss \u001b[1;36m1.9881\u001b[0m, time \u001b[1;36m4592.88\u001b[0m ms, mfu \u001b[1;36m1.59\u001b[0m%\n",
            "Running command: python3 train.py --max_iters 500 --n_layer 6 --n_head 6 --n_embd 384 --block_size 64 --device cuda --dtype float16 --dataset shakespeare_char --no-use_rotary_embeddings --use_abs_pos_embeddings --tensorboard_run_name config-500-6-6-384-64-cuda-float16-shakespeare_char-False-True-h --out_dir out_test/20240120_002817_config-500-6-6-384-64-cuda-float16-shakespeare_char-False-True-h\n",
            "2024-01-20 00:28:19.199711: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-01-20 00:28:19.199769: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-01-20 00:28:19.201091: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-01-20 00:28:19.208151: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-01-20 00:28:20.601905: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "\u001b[1m{\u001b[0m\n",
            "    \u001b[32m'block_size'\u001b[0m: \u001b[1;36m64\u001b[0m,\n",
            "    \u001b[32m'n_layer'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_head'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_embd'\u001b[0m: \u001b[1;36m384\u001b[0m,\n",
            "    \u001b[32m'dropout'\u001b[0m: \u001b[1;36m0.2\u001b[0m,\n",
            "    \u001b[32m'use_post_ln'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'layernorm_variant'\u001b[0m: \u001b[32m'rmsnorm'\u001b[0m,\n",
            "    \u001b[32m'bias'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'activation_variant'\u001b[0m: \u001b[32m'gelu'\u001b[0m,\n",
            "    \u001b[32m'use_rotary_embeddings'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'rope_variant'\u001b[0m: \u001b[32m'rope'\u001b[0m,\n",
            "    \u001b[32m'shortrope_length'\u001b[0m: \u001b[1;36m16\u001b[0m,\n",
            "    \u001b[32m'use_abs_pos_embeddings'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_attn'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_output'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_beta'\u001b[0m: \u001b[1;36m0.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_gamma'\u001b[0m: \u001b[1;36m100.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_use_euler_base'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'constantmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_x_intercept'\u001b[0m: \u001b[1;36m-100.0\u001b[0m,\n",
            "    \u001b[32m'polymax_y_intercept'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n",
            "    \u001b[32m'polymax_power'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_divisor'\u001b[0m: \u001b[1;36m1000.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_use_euler_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'strongermax_strength'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'softermax_use_xmax'\u001b[0m: \u001b[3;92mTrue\u001b[0m\n",
            "\u001b[1m}\u001b[0m\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "number of parameters: 10.65M\n",
            "num decayed parameter tensors: 26, with 10,666,368 parameters\n",
            "num non-decayed parameter tensors: 13, with 4,992 parameters\n",
            "using fused AdamW: True\n",
            "step \u001b[1;36m0\u001b[0m: train loss \u001b[1;36m4.3242\u001b[0m, val loss \u001b[1;36m4.3190\u001b[0m\n",
            "iter \u001b[1;36m0\u001b[0m: loss \u001b[1;36m4.3068\u001b[0m, time \u001b[1;36m5067.73\u001b[0m ms, mfu \u001b[1;36m-100.00\u001b[0m%\n",
            "iter \u001b[1;36m10\u001b[0m: loss \u001b[1;36m3.3844\u001b[0m, time \u001b[1;36m49.59\u001b[0m ms, mfu \u001b[1;36m1.74\u001b[0m%\n",
            "iter \u001b[1;36m20\u001b[0m: loss \u001b[1;36m3.1978\u001b[0m, time \u001b[1;36m48.97\u001b[0m ms, mfu \u001b[1;36m1.74\u001b[0m%\n",
            "iter \u001b[1;36m30\u001b[0m: loss \u001b[1;36m2.9954\u001b[0m, time \u001b[1;36m49.42\u001b[0m ms, mfu \u001b[1;36m1.74\u001b[0m%\n",
            "iter \u001b[1;36m40\u001b[0m: loss \u001b[1;36m2.8807\u001b[0m, time \u001b[1;36m49.21\u001b[0m ms, mfu \u001b[1;36m1.74\u001b[0m%\n",
            "iter \u001b[1;36m50\u001b[0m: loss \u001b[1;36m2.7464\u001b[0m, time \u001b[1;36m50.06\u001b[0m ms, mfu \u001b[1;36m1.74\u001b[0m%\n",
            "iter \u001b[1;36m60\u001b[0m: loss \u001b[1;36m2.6839\u001b[0m, time \u001b[1;36m49.84\u001b[0m ms, mfu \u001b[1;36m1.74\u001b[0m%\n",
            "iter \u001b[1;36m70\u001b[0m: loss \u001b[1;36m2.6746\u001b[0m, time \u001b[1;36m48.94\u001b[0m ms, mfu \u001b[1;36m1.74\u001b[0m%\n",
            "iter \u001b[1;36m80\u001b[0m: loss \u001b[1;36m2.6093\u001b[0m, time \u001b[1;36m49.96\u001b[0m ms, mfu \u001b[1;36m1.74\u001b[0m%\n",
            "iter \u001b[1;36m90\u001b[0m: loss \u001b[1;36m2.5877\u001b[0m, time \u001b[1;36m49.05\u001b[0m ms, mfu \u001b[1;36m1.74\u001b[0m%\n",
            "iter \u001b[1;36m100\u001b[0m: loss \u001b[1;36m2.5484\u001b[0m, time \u001b[1;36m49.76\u001b[0m ms, mfu \u001b[1;36m1.74\u001b[0m%\n",
            "iter \u001b[1;36m110\u001b[0m: loss \u001b[1;36m2.5027\u001b[0m, time \u001b[1;36m48.73\u001b[0m ms, mfu \u001b[1;36m1.74\u001b[0m%\n",
            "iter \u001b[1;36m120\u001b[0m: loss \u001b[1;36m2.5362\u001b[0m, time \u001b[1;36m47.69\u001b[0m ms, mfu \u001b[1;36m1.75\u001b[0m%\n",
            "iter \u001b[1;36m130\u001b[0m: loss \u001b[1;36m2.4874\u001b[0m, time \u001b[1;36m49.73\u001b[0m ms, mfu \u001b[1;36m1.75\u001b[0m%\n",
            "iter \u001b[1;36m140\u001b[0m: loss \u001b[1;36m2.4893\u001b[0m, time \u001b[1;36m49.55\u001b[0m ms, mfu \u001b[1;36m1.75\u001b[0m%\n",
            "iter \u001b[1;36m150\u001b[0m: loss \u001b[1;36m2.4143\u001b[0m, time \u001b[1;36m50.55\u001b[0m ms, mfu \u001b[1;36m1.74\u001b[0m%\n",
            "iter \u001b[1;36m160\u001b[0m: loss \u001b[1;36m2.4283\u001b[0m, time \u001b[1;36m49.70\u001b[0m ms, mfu \u001b[1;36m1.74\u001b[0m%\n",
            "iter \u001b[1;36m170\u001b[0m: loss \u001b[1;36m2.4257\u001b[0m, time \u001b[1;36m48.82\u001b[0m ms, mfu \u001b[1;36m1.74\u001b[0m%\n",
            "iter \u001b[1;36m180\u001b[0m: loss \u001b[1;36m2.3791\u001b[0m, time \u001b[1;36m49.58\u001b[0m ms, mfu \u001b[1;36m1.74\u001b[0m%\n",
            "iter \u001b[1;36m190\u001b[0m: loss \u001b[1;36m2.3323\u001b[0m, time \u001b[1;36m48.00\u001b[0m ms, mfu \u001b[1;36m1.75\u001b[0m%\n",
            "iter \u001b[1;36m200\u001b[0m: loss \u001b[1;36m2.3424\u001b[0m, time \u001b[1;36m48.98\u001b[0m ms, mfu \u001b[1;36m1.75\u001b[0m%\n",
            "iter \u001b[1;36m210\u001b[0m: loss \u001b[1;36m2.3257\u001b[0m, time \u001b[1;36m50.12\u001b[0m ms, mfu \u001b[1;36m1.75\u001b[0m%\n",
            "iter \u001b[1;36m220\u001b[0m: loss \u001b[1;36m2.3630\u001b[0m, time \u001b[1;36m48.70\u001b[0m ms, mfu \u001b[1;36m1.75\u001b[0m%\n",
            "iter \u001b[1;36m230\u001b[0m: loss \u001b[1;36m2.3269\u001b[0m, time \u001b[1;36m48.57\u001b[0m ms, mfu \u001b[1;36m1.75\u001b[0m%\n",
            "iter \u001b[1;36m240\u001b[0m: loss \u001b[1;36m2.2751\u001b[0m, time \u001b[1;36m51.05\u001b[0m ms, mfu \u001b[1;36m1.75\u001b[0m%\n",
            "step \u001b[1;36m250\u001b[0m: train loss \u001b[1;36m2.1997\u001b[0m, val loss \u001b[1;36m2.2277\u001b[0m\n",
            "saving checkpoint to \n",
            "out_test/20240120_002817_config-\u001b[1;36m500\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m384\u001b[0m-\u001b[1;36m64\u001b[0m-cuda-float16-shakespeare_char-\u001b[3;91mFalse\u001b[0m-\u001b[3;92mTrue\u001b[0m-h\n",
            "iter \u001b[1;36m250\u001b[0m: loss \u001b[1;36m2.2827\u001b[0m, time \u001b[1;36m4919.15\u001b[0m ms, mfu \u001b[1;36m1.57\u001b[0m%\n",
            "iter \u001b[1;36m260\u001b[0m: loss \u001b[1;36m2.2728\u001b[0m, time \u001b[1;36m49.81\u001b[0m ms, mfu \u001b[1;36m1.59\u001b[0m%\n",
            "iter \u001b[1;36m270\u001b[0m: loss \u001b[1;36m2.2380\u001b[0m, time \u001b[1;36m50.74\u001b[0m ms, mfu \u001b[1;36m1.60\u001b[0m%\n",
            "iter \u001b[1;36m280\u001b[0m: loss \u001b[1;36m2.2038\u001b[0m, time \u001b[1;36m50.89\u001b[0m ms, mfu \u001b[1;36m1.61\u001b[0m%\n",
            "iter \u001b[1;36m290\u001b[0m: loss \u001b[1;36m2.2445\u001b[0m, time \u001b[1;36m51.63\u001b[0m ms, mfu \u001b[1;36m1.61\u001b[0m%\n",
            "iter \u001b[1;36m300\u001b[0m: loss \u001b[1;36m2.2033\u001b[0m, time \u001b[1;36m49.02\u001b[0m ms, mfu \u001b[1;36m1.63\u001b[0m%\n",
            "iter \u001b[1;36m310\u001b[0m: loss \u001b[1;36m2.2223\u001b[0m, time \u001b[1;36m50.44\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "iter \u001b[1;36m320\u001b[0m: loss \u001b[1;36m2.1843\u001b[0m, time \u001b[1;36m49.73\u001b[0m ms, mfu \u001b[1;36m1.65\u001b[0m%\n",
            "iter \u001b[1;36m330\u001b[0m: loss \u001b[1;36m2.1800\u001b[0m, time \u001b[1;36m49.72\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m340\u001b[0m: loss \u001b[1;36m2.1070\u001b[0m, time \u001b[1;36m50.03\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m350\u001b[0m: loss \u001b[1;36m2.1371\u001b[0m, time \u001b[1;36m50.99\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m360\u001b[0m: loss \u001b[1;36m2.1435\u001b[0m, time \u001b[1;36m50.75\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m370\u001b[0m: loss \u001b[1;36m2.1212\u001b[0m, time \u001b[1;36m50.96\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m380\u001b[0m: loss \u001b[1;36m2.1330\u001b[0m, time \u001b[1;36m49.21\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m390\u001b[0m: loss \u001b[1;36m2.0787\u001b[0m, time \u001b[1;36m49.12\u001b[0m ms, mfu \u001b[1;36m1.69\u001b[0m%\n",
            "iter \u001b[1;36m400\u001b[0m: loss \u001b[1;36m2.1050\u001b[0m, time \u001b[1;36m49.59\u001b[0m ms, mfu \u001b[1;36m1.69\u001b[0m%\n",
            "iter \u001b[1;36m410\u001b[0m: loss \u001b[1;36m2.0801\u001b[0m, time \u001b[1;36m49.81\u001b[0m ms, mfu \u001b[1;36m1.70\u001b[0m%\n",
            "iter \u001b[1;36m420\u001b[0m: loss \u001b[1;36m2.0883\u001b[0m, time \u001b[1;36m49.81\u001b[0m ms, mfu \u001b[1;36m1.70\u001b[0m%\n",
            "iter \u001b[1;36m430\u001b[0m: loss \u001b[1;36m2.0279\u001b[0m, time \u001b[1;36m50.47\u001b[0m ms, mfu \u001b[1;36m1.70\u001b[0m%\n",
            "iter \u001b[1;36m440\u001b[0m: loss \u001b[1;36m2.0148\u001b[0m, time \u001b[1;36m50.08\u001b[0m ms, mfu \u001b[1;36m1.70\u001b[0m%\n",
            "iter \u001b[1;36m450\u001b[0m: loss \u001b[1;36m2.0287\u001b[0m, time \u001b[1;36m50.32\u001b[0m ms, mfu \u001b[1;36m1.70\u001b[0m%\n",
            "iter \u001b[1;36m460\u001b[0m: loss \u001b[1;36m2.0280\u001b[0m, time \u001b[1;36m49.75\u001b[0m ms, mfu \u001b[1;36m1.71\u001b[0m%\n",
            "iter \u001b[1;36m470\u001b[0m: loss \u001b[1;36m1.9884\u001b[0m, time \u001b[1;36m53.03\u001b[0m ms, mfu \u001b[1;36m1.70\u001b[0m%\n",
            "iter \u001b[1;36m480\u001b[0m: loss \u001b[1;36m2.0140\u001b[0m, time \u001b[1;36m52.25\u001b[0m ms, mfu \u001b[1;36m1.69\u001b[0m%\n",
            "iter \u001b[1;36m490\u001b[0m: loss \u001b[1;36m2.0285\u001b[0m, time \u001b[1;36m51.01\u001b[0m ms, mfu \u001b[1;36m1.69\u001b[0m%\n",
            "step \u001b[1;36m500\u001b[0m: train loss \u001b[1;36m1.8897\u001b[0m, val loss \u001b[1;36m1.9912\u001b[0m\n",
            "saving checkpoint to \n",
            "out_test/20240120_002817_config-\u001b[1;36m500\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m384\u001b[0m-\u001b[1;36m64\u001b[0m-cuda-float16-shakespeare_char-\u001b[3;91mFalse\u001b[0m-\u001b[3;92mTrue\u001b[0m-h\n",
            "iter \u001b[1;36m500\u001b[0m: loss \u001b[1;36m1.9881\u001b[0m, time \u001b[1;36m5465.05\u001b[0m ms, mfu \u001b[1;36m1.52\u001b[0m%\n",
            "Running command: python3 train.py --max_iters 500 --n_layer 6 --n_head 6 --n_embd 384 --block_size 64 --device cuda --dtype float16 --dataset shakespeare_char --no-use_rotary_embeddings --use_abs_pos_embeddings --tensorboard_run_name config-500-6-6-384-64-cuda-float16-shakespeare_char-False-True-a --out_dir out_test/20240120_002905_config-500-6-6-384-64-cuda-float16-shakespeare_char-False-True-a\n",
            "2024-01-20 00:29:06.960137: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-01-20 00:29:06.960191: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-01-20 00:29:06.961465: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-01-20 00:29:06.968384: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-01-20 00:29:07.955048: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "\u001b[1m{\u001b[0m\n",
            "    \u001b[32m'block_size'\u001b[0m: \u001b[1;36m64\u001b[0m,\n",
            "    \u001b[32m'n_layer'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_head'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_embd'\u001b[0m: \u001b[1;36m384\u001b[0m,\n",
            "    \u001b[32m'dropout'\u001b[0m: \u001b[1;36m0.2\u001b[0m,\n",
            "    \u001b[32m'use_post_ln'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'layernorm_variant'\u001b[0m: \u001b[32m'rmsnorm'\u001b[0m,\n",
            "    \u001b[32m'bias'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'activation_variant'\u001b[0m: \u001b[32m'gelu'\u001b[0m,\n",
            "    \u001b[32m'use_rotary_embeddings'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'rope_variant'\u001b[0m: \u001b[32m'rope'\u001b[0m,\n",
            "    \u001b[32m'shortrope_length'\u001b[0m: \u001b[1;36m16\u001b[0m,\n",
            "    \u001b[32m'use_abs_pos_embeddings'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_attn'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_output'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_beta'\u001b[0m: \u001b[1;36m0.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_gamma'\u001b[0m: \u001b[1;36m100.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_use_euler_base'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'constantmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_x_intercept'\u001b[0m: \u001b[1;36m-100.0\u001b[0m,\n",
            "    \u001b[32m'polymax_y_intercept'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n",
            "    \u001b[32m'polymax_power'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_divisor'\u001b[0m: \u001b[1;36m1000.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_use_euler_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'strongermax_strength'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'softermax_use_xmax'\u001b[0m: \u001b[3;92mTrue\u001b[0m\n",
            "\u001b[1m}\u001b[0m\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "number of parameters: 10.65M\n",
            "num decayed parameter tensors: 26, with 10,666,368 parameters\n",
            "num non-decayed parameter tensors: 13, with 4,992 parameters\n",
            "using fused AdamW: True\n",
            "step \u001b[1;36m0\u001b[0m: train loss \u001b[1;36m4.3242\u001b[0m, val loss \u001b[1;36m4.3190\u001b[0m\n",
            "iter \u001b[1;36m0\u001b[0m: loss \u001b[1;36m4.3068\u001b[0m, time \u001b[1;36m5489.76\u001b[0m ms, mfu \u001b[1;36m-100.00\u001b[0m%\n",
            "iter \u001b[1;36m10\u001b[0m: loss \u001b[1;36m3.3844\u001b[0m, time \u001b[1;36m49.92\u001b[0m ms, mfu \u001b[1;36m1.73\u001b[0m%\n",
            "iter \u001b[1;36m20\u001b[0m: loss \u001b[1;36m3.1978\u001b[0m, time \u001b[1;36m51.71\u001b[0m ms, mfu \u001b[1;36m1.72\u001b[0m%\n",
            "iter \u001b[1;36m30\u001b[0m: loss \u001b[1;36m2.9954\u001b[0m, time \u001b[1;36m51.39\u001b[0m ms, mfu \u001b[1;36m1.72\u001b[0m%\n",
            "iter \u001b[1;36m40\u001b[0m: loss \u001b[1;36m2.8807\u001b[0m, time \u001b[1;36m52.37\u001b[0m ms, mfu \u001b[1;36m1.71\u001b[0m%\n",
            "iter \u001b[1;36m50\u001b[0m: loss \u001b[1;36m2.7464\u001b[0m, time \u001b[1;36m52.49\u001b[0m ms, mfu \u001b[1;36m1.70\u001b[0m%\n",
            "iter \u001b[1;36m60\u001b[0m: loss \u001b[1;36m2.6839\u001b[0m, time \u001b[1;36m50.99\u001b[0m ms, mfu \u001b[1;36m1.70\u001b[0m%\n",
            "iter \u001b[1;36m70\u001b[0m: loss \u001b[1;36m2.6746\u001b[0m, time \u001b[1;36m52.29\u001b[0m ms, mfu \u001b[1;36m1.70\u001b[0m%\n",
            "iter \u001b[1;36m80\u001b[0m: loss \u001b[1;36m2.6093\u001b[0m, time \u001b[1;36m50.62\u001b[0m ms, mfu \u001b[1;36m1.70\u001b[0m%\n",
            "iter \u001b[1;36m90\u001b[0m: loss \u001b[1;36m2.5877\u001b[0m, time \u001b[1;36m52.48\u001b[0m ms, mfu \u001b[1;36m1.69\u001b[0m%\n",
            "iter \u001b[1;36m100\u001b[0m: loss \u001b[1;36m2.5484\u001b[0m, time \u001b[1;36m52.83\u001b[0m ms, mfu \u001b[1;36m1.69\u001b[0m%\n",
            "iter \u001b[1;36m110\u001b[0m: loss \u001b[1;36m2.5027\u001b[0m, time \u001b[1;36m52.44\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m120\u001b[0m: loss \u001b[1;36m2.5362\u001b[0m, time \u001b[1;36m50.59\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m130\u001b[0m: loss \u001b[1;36m2.4874\u001b[0m, time \u001b[1;36m52.73\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m140\u001b[0m: loss \u001b[1;36m2.4893\u001b[0m, time \u001b[1;36m51.72\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m150\u001b[0m: loss \u001b[1;36m2.4143\u001b[0m, time \u001b[1;36m52.12\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m160\u001b[0m: loss \u001b[1;36m2.4283\u001b[0m, time \u001b[1;36m51.86\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m170\u001b[0m: loss \u001b[1;36m2.4257\u001b[0m, time \u001b[1;36m52.99\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m180\u001b[0m: loss \u001b[1;36m2.3791\u001b[0m, time \u001b[1;36m52.17\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m190\u001b[0m: loss \u001b[1;36m2.3323\u001b[0m, time \u001b[1;36m52.23\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m200\u001b[0m: loss \u001b[1;36m2.3424\u001b[0m, time \u001b[1;36m54.16\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m210\u001b[0m: loss \u001b[1;36m2.3257\u001b[0m, time \u001b[1;36m52.80\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m220\u001b[0m: loss \u001b[1;36m2.3630\u001b[0m, time \u001b[1;36m51.06\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m230\u001b[0m: loss \u001b[1;36m2.3269\u001b[0m, time \u001b[1;36m51.49\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m240\u001b[0m: loss \u001b[1;36m2.2751\u001b[0m, time \u001b[1;36m51.75\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "step \u001b[1;36m250\u001b[0m: train loss \u001b[1;36m2.1997\u001b[0m, val loss \u001b[1;36m2.2277\u001b[0m\n",
            "saving checkpoint to \n",
            "out_test/20240120_002905_config-\u001b[1;36m500\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m384\u001b[0m-\u001b[1;36m64\u001b[0m-cuda-float16-shakespeare_char-\u001b[3;91mFalse\u001b[0m-\u001b[3;92mTrue\u001b[0m-a\n",
            "iter \u001b[1;36m250\u001b[0m: loss \u001b[1;36m2.2827\u001b[0m, time \u001b[1;36m4801.84\u001b[0m ms, mfu \u001b[1;36m1.50\u001b[0m%\n",
            "iter \u001b[1;36m260\u001b[0m: loss \u001b[1;36m2.2728\u001b[0m, time \u001b[1;36m52.96\u001b[0m ms, mfu \u001b[1;36m1.51\u001b[0m%\n",
            "iter \u001b[1;36m270\u001b[0m: loss \u001b[1;36m2.2380\u001b[0m, time \u001b[1;36m53.71\u001b[0m ms, mfu \u001b[1;36m1.52\u001b[0m%\n",
            "iter \u001b[1;36m280\u001b[0m: loss \u001b[1;36m2.2038\u001b[0m, time \u001b[1;36m54.10\u001b[0m ms, mfu \u001b[1;36m1.53\u001b[0m%\n",
            "iter \u001b[1;36m290\u001b[0m: loss \u001b[1;36m2.2445\u001b[0m, time \u001b[1;36m53.24\u001b[0m ms, mfu \u001b[1;36m1.54\u001b[0m%\n",
            "iter \u001b[1;36m300\u001b[0m: loss \u001b[1;36m2.2033\u001b[0m, time \u001b[1;36m53.67\u001b[0m ms, mfu \u001b[1;36m1.54\u001b[0m%\n",
            "iter \u001b[1;36m310\u001b[0m: loss \u001b[1;36m2.2223\u001b[0m, time \u001b[1;36m53.02\u001b[0m ms, mfu \u001b[1;36m1.55\u001b[0m%\n",
            "iter \u001b[1;36m320\u001b[0m: loss \u001b[1;36m2.1843\u001b[0m, time \u001b[1;36m53.42\u001b[0m ms, mfu \u001b[1;36m1.56\u001b[0m%\n",
            "iter \u001b[1;36m330\u001b[0m: loss \u001b[1;36m2.1800\u001b[0m, time \u001b[1;36m51.23\u001b[0m ms, mfu \u001b[1;36m1.57\u001b[0m%\n",
            "iter \u001b[1;36m340\u001b[0m: loss \u001b[1;36m2.1070\u001b[0m, time \u001b[1;36m51.79\u001b[0m ms, mfu \u001b[1;36m1.58\u001b[0m%\n",
            "iter \u001b[1;36m350\u001b[0m: loss \u001b[1;36m2.1371\u001b[0m, time \u001b[1;36m53.74\u001b[0m ms, mfu \u001b[1;36m1.58\u001b[0m%\n",
            "iter \u001b[1;36m360\u001b[0m: loss \u001b[1;36m2.1435\u001b[0m, time \u001b[1;36m53.51\u001b[0m ms, mfu \u001b[1;36m1.58\u001b[0m%\n",
            "iter \u001b[1;36m370\u001b[0m: loss \u001b[1;36m2.1212\u001b[0m, time \u001b[1;36m52.31\u001b[0m ms, mfu \u001b[1;36m1.59\u001b[0m%\n",
            "iter \u001b[1;36m380\u001b[0m: loss \u001b[1;36m2.1330\u001b[0m, time \u001b[1;36m51.80\u001b[0m ms, mfu \u001b[1;36m1.60\u001b[0m%\n",
            "iter \u001b[1;36m390\u001b[0m: loss \u001b[1;36m2.0787\u001b[0m, time \u001b[1;36m53.57\u001b[0m ms, mfu \u001b[1;36m1.60\u001b[0m%\n",
            "iter \u001b[1;36m400\u001b[0m: loss \u001b[1;36m2.1050\u001b[0m, time \u001b[1;36m53.75\u001b[0m ms, mfu \u001b[1;36m1.60\u001b[0m%\n",
            "iter \u001b[1;36m410\u001b[0m: loss \u001b[1;36m2.0801\u001b[0m, time \u001b[1;36m53.65\u001b[0m ms, mfu \u001b[1;36m1.60\u001b[0m%\n",
            "iter \u001b[1;36m420\u001b[0m: loss \u001b[1;36m2.0883\u001b[0m, time \u001b[1;36m52.29\u001b[0m ms, mfu \u001b[1;36m1.61\u001b[0m%\n",
            "iter \u001b[1;36m430\u001b[0m: loss \u001b[1;36m2.0279\u001b[0m, time \u001b[1;36m51.80\u001b[0m ms, mfu \u001b[1;36m1.61\u001b[0m%\n",
            "iter \u001b[1;36m440\u001b[0m: loss \u001b[1;36m2.0148\u001b[0m, time \u001b[1;36m49.59\u001b[0m ms, mfu \u001b[1;36m1.62\u001b[0m%\n",
            "iter \u001b[1;36m450\u001b[0m: loss \u001b[1;36m2.0287\u001b[0m, time \u001b[1;36m52.74\u001b[0m ms, mfu \u001b[1;36m1.62\u001b[0m%\n",
            "iter \u001b[1;36m460\u001b[0m: loss \u001b[1;36m2.0280\u001b[0m, time \u001b[1;36m54.07\u001b[0m ms, mfu \u001b[1;36m1.62\u001b[0m%\n",
            "iter \u001b[1;36m470\u001b[0m: loss \u001b[1;36m1.9884\u001b[0m, time \u001b[1;36m53.56\u001b[0m ms, mfu \u001b[1;36m1.62\u001b[0m%\n",
            "iter \u001b[1;36m480\u001b[0m: loss \u001b[1;36m2.0140\u001b[0m, time \u001b[1;36m51.92\u001b[0m ms, mfu \u001b[1;36m1.62\u001b[0m%\n",
            "iter \u001b[1;36m490\u001b[0m: loss \u001b[1;36m2.0285\u001b[0m, time \u001b[1;36m51.74\u001b[0m ms, mfu \u001b[1;36m1.63\u001b[0m%\n",
            "step \u001b[1;36m500\u001b[0m: train loss \u001b[1;36m1.8897\u001b[0m, val loss \u001b[1;36m1.9912\u001b[0m\n",
            "saving checkpoint to \n",
            "out_test/20240120_002905_config-\u001b[1;36m500\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m384\u001b[0m-\u001b[1;36m64\u001b[0m-cuda-float16-shakespeare_char-\u001b[3;91mFalse\u001b[0m-\u001b[3;92mTrue\u001b[0m-a\n",
            "iter \u001b[1;36m500\u001b[0m: loss \u001b[1;36m1.9881\u001b[0m, time \u001b[1;36m5128.70\u001b[0m ms, mfu \u001b[1;36m1.47\u001b[0m%\n",
            "Running command: python3 train.py --max_iters 500 --n_layer 6 --n_head 6 --n_embd 384 --block_size 64 --device cuda --dtype float16 --dataset shakespeare_char --no-use_rotary_embeddings --use_abs_pos_embeddings --tensorboard_run_name config-500-6-6-384-64-cuda-float16-shakespeare_char-False-True-k --out_dir out_test/20240120_002953_config-500-6-6-384-64-cuda-float16-shakespeare_char-False-True-k\n",
            "2024-01-20 00:29:55.260295: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-01-20 00:29:55.260349: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-01-20 00:29:55.262140: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-01-20 00:29:55.272493: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-01-20 00:29:56.252849: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "\u001b[1m{\u001b[0m\n",
            "    \u001b[32m'block_size'\u001b[0m: \u001b[1;36m64\u001b[0m,\n",
            "    \u001b[32m'n_layer'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_head'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_embd'\u001b[0m: \u001b[1;36m384\u001b[0m,\n",
            "    \u001b[32m'dropout'\u001b[0m: \u001b[1;36m0.2\u001b[0m,\n",
            "    \u001b[32m'use_post_ln'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'layernorm_variant'\u001b[0m: \u001b[32m'rmsnorm'\u001b[0m,\n",
            "    \u001b[32m'bias'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'activation_variant'\u001b[0m: \u001b[32m'gelu'\u001b[0m,\n",
            "    \u001b[32m'use_rotary_embeddings'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'rope_variant'\u001b[0m: \u001b[32m'rope'\u001b[0m,\n",
            "    \u001b[32m'shortrope_length'\u001b[0m: \u001b[1;36m16\u001b[0m,\n",
            "    \u001b[32m'use_abs_pos_embeddings'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_attn'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_output'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_beta'\u001b[0m: \u001b[1;36m0.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_gamma'\u001b[0m: \u001b[1;36m100.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_use_euler_base'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'constantmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_x_intercept'\u001b[0m: \u001b[1;36m-100.0\u001b[0m,\n",
            "    \u001b[32m'polymax_y_intercept'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n",
            "    \u001b[32m'polymax_power'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_divisor'\u001b[0m: \u001b[1;36m1000.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_use_euler_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'strongermax_strength'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'softermax_use_xmax'\u001b[0m: \u001b[3;92mTrue\u001b[0m\n",
            "\u001b[1m}\u001b[0m\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "number of parameters: 10.65M\n",
            "num decayed parameter tensors: 26, with 10,666,368 parameters\n",
            "num non-decayed parameter tensors: 13, with 4,992 parameters\n",
            "using fused AdamW: True\n",
            "step \u001b[1;36m0\u001b[0m: train loss \u001b[1;36m4.3242\u001b[0m, val loss \u001b[1;36m4.3190\u001b[0m\n",
            "iter \u001b[1;36m0\u001b[0m: loss \u001b[1;36m4.3068\u001b[0m, time \u001b[1;36m5554.61\u001b[0m ms, mfu \u001b[1;36m-100.00\u001b[0m%\n",
            "iter \u001b[1;36m10\u001b[0m: loss \u001b[1;36m3.3844\u001b[0m, time \u001b[1;36m49.59\u001b[0m ms, mfu \u001b[1;36m1.74\u001b[0m%\n",
            "iter \u001b[1;36m20\u001b[0m: loss \u001b[1;36m3.1978\u001b[0m, time \u001b[1;36m51.82\u001b[0m ms, mfu \u001b[1;36m1.73\u001b[0m%\n",
            "iter \u001b[1;36m30\u001b[0m: loss \u001b[1;36m2.9954\u001b[0m, time \u001b[1;36m51.85\u001b[0m ms, mfu \u001b[1;36m1.72\u001b[0m%\n",
            "iter \u001b[1;36m40\u001b[0m: loss \u001b[1;36m2.8807\u001b[0m, time \u001b[1;36m50.03\u001b[0m ms, mfu \u001b[1;36m1.72\u001b[0m%\n",
            "iter \u001b[1;36m50\u001b[0m: loss \u001b[1;36m2.7464\u001b[0m, time \u001b[1;36m52.22\u001b[0m ms, mfu \u001b[1;36m1.72\u001b[0m%\n",
            "iter \u001b[1;36m60\u001b[0m: loss \u001b[1;36m2.6839\u001b[0m, time \u001b[1;36m50.81\u001b[0m ms, mfu \u001b[1;36m1.71\u001b[0m%\n",
            "iter \u001b[1;36m70\u001b[0m: loss \u001b[1;36m2.6746\u001b[0m, time \u001b[1;36m52.02\u001b[0m ms, mfu \u001b[1;36m1.71\u001b[0m%\n",
            "iter \u001b[1;36m80\u001b[0m: loss \u001b[1;36m2.6093\u001b[0m, time \u001b[1;36m51.04\u001b[0m ms, mfu \u001b[1;36m1.71\u001b[0m%\n",
            "iter \u001b[1;36m90\u001b[0m: loss \u001b[1;36m2.5877\u001b[0m, time \u001b[1;36m52.04\u001b[0m ms, mfu \u001b[1;36m1.70\u001b[0m%\n",
            "iter \u001b[1;36m100\u001b[0m: loss \u001b[1;36m2.5484\u001b[0m, time \u001b[1;36m52.04\u001b[0m ms, mfu \u001b[1;36m1.70\u001b[0m%\n",
            "iter \u001b[1;36m110\u001b[0m: loss \u001b[1;36m2.5027\u001b[0m, time \u001b[1;36m52.54\u001b[0m ms, mfu \u001b[1;36m1.69\u001b[0m%\n",
            "iter \u001b[1;36m120\u001b[0m: loss \u001b[1;36m2.5362\u001b[0m, time \u001b[1;36m52.45\u001b[0m ms, mfu \u001b[1;36m1.69\u001b[0m%\n",
            "iter \u001b[1;36m130\u001b[0m: loss \u001b[1;36m2.4874\u001b[0m, time \u001b[1;36m51.12\u001b[0m ms, mfu \u001b[1;36m1.69\u001b[0m%\n",
            "iter \u001b[1;36m140\u001b[0m: loss \u001b[1;36m2.4893\u001b[0m, time \u001b[1;36m52.49\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m150\u001b[0m: loss \u001b[1;36m2.4143\u001b[0m, time \u001b[1;36m50.73\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m160\u001b[0m: loss \u001b[1;36m2.4283\u001b[0m, time \u001b[1;36m51.60\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m170\u001b[0m: loss \u001b[1;36m2.4257\u001b[0m, time \u001b[1;36m52.10\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m180\u001b[0m: loss \u001b[1;36m2.3791\u001b[0m, time \u001b[1;36m51.86\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m190\u001b[0m: loss \u001b[1;36m2.3323\u001b[0m, time \u001b[1;36m52.57\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m200\u001b[0m: loss \u001b[1;36m2.3424\u001b[0m, time \u001b[1;36m51.82\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m210\u001b[0m: loss \u001b[1;36m2.3257\u001b[0m, time \u001b[1;36m51.98\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m220\u001b[0m: loss \u001b[1;36m2.3630\u001b[0m, time \u001b[1;36m53.41\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m230\u001b[0m: loss \u001b[1;36m2.3269\u001b[0m, time \u001b[1;36m51.44\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m240\u001b[0m: loss \u001b[1;36m2.2751\u001b[0m, time \u001b[1;36m52.00\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "step \u001b[1;36m250\u001b[0m: train loss \u001b[1;36m2.1997\u001b[0m, val loss \u001b[1;36m2.2277\u001b[0m\n",
            "saving checkpoint to \n",
            "out_test/20240120_002953_config-\u001b[1;36m500\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m384\u001b[0m-\u001b[1;36m64\u001b[0m-cuda-float16-shakespeare_char-\u001b[3;91mFalse\u001b[0m-\u001b[3;92mTrue\u001b[0m-k\n",
            "iter \u001b[1;36m250\u001b[0m: loss \u001b[1;36m2.2827\u001b[0m, time \u001b[1;36m4944.26\u001b[0m ms, mfu \u001b[1;36m1.50\u001b[0m%\n",
            "iter \u001b[1;36m260\u001b[0m: loss \u001b[1;36m2.2728\u001b[0m, time \u001b[1;36m50.85\u001b[0m ms, mfu \u001b[1;36m1.52\u001b[0m%\n",
            "iter \u001b[1;36m270\u001b[0m: loss \u001b[1;36m2.2380\u001b[0m, time \u001b[1;36m50.74\u001b[0m ms, mfu \u001b[1;36m1.54\u001b[0m%\n",
            "iter \u001b[1;36m280\u001b[0m: loss \u001b[1;36m2.2038\u001b[0m, time \u001b[1;36m51.99\u001b[0m ms, mfu \u001b[1;36m1.55\u001b[0m%\n",
            "iter \u001b[1;36m290\u001b[0m: loss \u001b[1;36m2.2445\u001b[0m, time \u001b[1;36m51.86\u001b[0m ms, mfu \u001b[1;36m1.56\u001b[0m%\n",
            "iter \u001b[1;36m300\u001b[0m: loss \u001b[1;36m2.2033\u001b[0m, time \u001b[1;36m51.65\u001b[0m ms, mfu \u001b[1;36m1.57\u001b[0m%\n",
            "iter \u001b[1;36m310\u001b[0m: loss \u001b[1;36m2.2223\u001b[0m, time \u001b[1;36m52.16\u001b[0m ms, mfu \u001b[1;36m1.58\u001b[0m%\n",
            "iter \u001b[1;36m320\u001b[0m: loss \u001b[1;36m2.1843\u001b[0m, time \u001b[1;36m52.14\u001b[0m ms, mfu \u001b[1;36m1.59\u001b[0m%\n",
            "iter \u001b[1;36m330\u001b[0m: loss \u001b[1;36m2.1800\u001b[0m, time \u001b[1;36m50.79\u001b[0m ms, mfu \u001b[1;36m1.60\u001b[0m%\n",
            "iter \u001b[1;36m340\u001b[0m: loss \u001b[1;36m2.1070\u001b[0m, time \u001b[1;36m51.38\u001b[0m ms, mfu \u001b[1;36m1.61\u001b[0m%\n",
            "iter \u001b[1;36m350\u001b[0m: loss \u001b[1;36m2.1371\u001b[0m, time \u001b[1;36m54.50\u001b[0m ms, mfu \u001b[1;36m1.60\u001b[0m%\n",
            "iter \u001b[1;36m360\u001b[0m: loss \u001b[1;36m2.1435\u001b[0m, time \u001b[1;36m53.02\u001b[0m ms, mfu \u001b[1;36m1.61\u001b[0m%\n",
            "iter \u001b[1;36m370\u001b[0m: loss \u001b[1;36m2.1212\u001b[0m, time \u001b[1;36m51.84\u001b[0m ms, mfu \u001b[1;36m1.61\u001b[0m%\n",
            "iter \u001b[1;36m380\u001b[0m: loss \u001b[1;36m2.1330\u001b[0m, time \u001b[1;36m51.20\u001b[0m ms, mfu \u001b[1;36m1.62\u001b[0m%\n",
            "iter \u001b[1;36m390\u001b[0m: loss \u001b[1;36m2.0787\u001b[0m, time \u001b[1;36m49.57\u001b[0m ms, mfu \u001b[1;36m1.63\u001b[0m%\n",
            "iter \u001b[1;36m400\u001b[0m: loss \u001b[1;36m2.1050\u001b[0m, time \u001b[1;36m52.25\u001b[0m ms, mfu \u001b[1;36m1.63\u001b[0m%\n",
            "iter \u001b[1;36m410\u001b[0m: loss \u001b[1;36m2.0801\u001b[0m, time \u001b[1;36m51.60\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "iter \u001b[1;36m420\u001b[0m: loss \u001b[1;36m2.0883\u001b[0m, time \u001b[1;36m52.88\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "iter \u001b[1;36m430\u001b[0m: loss \u001b[1;36m2.0279\u001b[0m, time \u001b[1;36m52.75\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "iter \u001b[1;36m440\u001b[0m: loss \u001b[1;36m2.0148\u001b[0m, time \u001b[1;36m52.22\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "iter \u001b[1;36m450\u001b[0m: loss \u001b[1;36m2.0287\u001b[0m, time \u001b[1;36m52.09\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "iter \u001b[1;36m460\u001b[0m: loss \u001b[1;36m2.0280\u001b[0m, time \u001b[1;36m51.08\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "iter \u001b[1;36m470\u001b[0m: loss \u001b[1;36m1.9884\u001b[0m, time \u001b[1;36m52.58\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "iter \u001b[1;36m480\u001b[0m: loss \u001b[1;36m2.0140\u001b[0m, time \u001b[1;36m52.95\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "iter \u001b[1;36m490\u001b[0m: loss \u001b[1;36m2.0285\u001b[0m, time \u001b[1;36m51.61\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "step \u001b[1;36m500\u001b[0m: train loss \u001b[1;36m1.8897\u001b[0m, val loss \u001b[1;36m1.9912\u001b[0m\n",
            "saving checkpoint to \n",
            "out_test/20240120_002953_config-\u001b[1;36m500\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m384\u001b[0m-\u001b[1;36m64\u001b[0m-cuda-float16-shakespeare_char-\u001b[3;91mFalse\u001b[0m-\u001b[3;92mTrue\u001b[0m-k\n",
            "iter \u001b[1;36m500\u001b[0m: loss \u001b[1;36m1.9881\u001b[0m, time \u001b[1;36m4956.09\u001b[0m ms, mfu \u001b[1;36m1.48\u001b[0m%\n",
            "Running command: python3 train.py --max_iters 500 --n_layer 6 --n_head 6 --n_embd 384 --block_size 64 --device cuda --dtype float16 --dataset shakespeare_char --no-use_rotary_embeddings --use_abs_pos_embeddings --tensorboard_run_name config-500-6-6-384-64-cuda-float16-shakespeare_char-False-True-e --out_dir out_test/20240120_003041_config-500-6-6-384-64-cuda-float16-shakespeare_char-False-True-e\n",
            "2024-01-20 00:30:43.515751: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-01-20 00:30:43.515807: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-01-20 00:30:43.517580: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-01-20 00:30:43.527758: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-01-20 00:30:45.000704: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "\u001b[1m{\u001b[0m\n",
            "    \u001b[32m'block_size'\u001b[0m: \u001b[1;36m64\u001b[0m,\n",
            "    \u001b[32m'n_layer'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_head'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_embd'\u001b[0m: \u001b[1;36m384\u001b[0m,\n",
            "    \u001b[32m'dropout'\u001b[0m: \u001b[1;36m0.2\u001b[0m,\n",
            "    \u001b[32m'use_post_ln'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'layernorm_variant'\u001b[0m: \u001b[32m'rmsnorm'\u001b[0m,\n",
            "    \u001b[32m'bias'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'activation_variant'\u001b[0m: \u001b[32m'gelu'\u001b[0m,\n",
            "    \u001b[32m'use_rotary_embeddings'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'rope_variant'\u001b[0m: \u001b[32m'rope'\u001b[0m,\n",
            "    \u001b[32m'shortrope_length'\u001b[0m: \u001b[1;36m16\u001b[0m,\n",
            "    \u001b[32m'use_abs_pos_embeddings'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_attn'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_output'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_beta'\u001b[0m: \u001b[1;36m0.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_gamma'\u001b[0m: \u001b[1;36m100.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_use_euler_base'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'constantmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_x_intercept'\u001b[0m: \u001b[1;36m-100.0\u001b[0m,\n",
            "    \u001b[32m'polymax_y_intercept'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n",
            "    \u001b[32m'polymax_power'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_divisor'\u001b[0m: \u001b[1;36m1000.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_use_euler_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'strongermax_strength'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'softermax_use_xmax'\u001b[0m: \u001b[3;92mTrue\u001b[0m\n",
            "\u001b[1m}\u001b[0m\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "number of parameters: 10.65M\n",
            "num decayed parameter tensors: 26, with 10,666,368 parameters\n",
            "num non-decayed parameter tensors: 13, with 4,992 parameters\n",
            "using fused AdamW: True\n",
            "step \u001b[1;36m0\u001b[0m: train loss \u001b[1;36m4.3242\u001b[0m, val loss \u001b[1;36m4.3190\u001b[0m\n",
            "iter \u001b[1;36m0\u001b[0m: loss \u001b[1;36m4.3068\u001b[0m, time \u001b[1;36m5155.56\u001b[0m ms, mfu \u001b[1;36m-100.00\u001b[0m%\n",
            "iter \u001b[1;36m10\u001b[0m: loss \u001b[1;36m3.3844\u001b[0m, time \u001b[1;36m51.37\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m20\u001b[0m: loss \u001b[1;36m3.1978\u001b[0m, time \u001b[1;36m51.66\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m30\u001b[0m: loss \u001b[1;36m2.9954\u001b[0m, time \u001b[1;36m50.65\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m40\u001b[0m: loss \u001b[1;36m2.8807\u001b[0m, time \u001b[1;36m52.11\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m50\u001b[0m: loss \u001b[1;36m2.7464\u001b[0m, time \u001b[1;36m51.03\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m60\u001b[0m: loss \u001b[1;36m2.6839\u001b[0m, time \u001b[1;36m50.66\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m70\u001b[0m: loss \u001b[1;36m2.6746\u001b[0m, time \u001b[1;36m52.34\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m80\u001b[0m: loss \u001b[1;36m2.6093\u001b[0m, time \u001b[1;36m51.88\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m90\u001b[0m: loss \u001b[1;36m2.5877\u001b[0m, time \u001b[1;36m52.94\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m100\u001b[0m: loss \u001b[1;36m2.5484\u001b[0m, time \u001b[1;36m53.45\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m110\u001b[0m: loss \u001b[1;36m2.5027\u001b[0m, time \u001b[1;36m52.58\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m120\u001b[0m: loss \u001b[1;36m2.5362\u001b[0m, time \u001b[1;36m51.68\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m130\u001b[0m: loss \u001b[1;36m2.4874\u001b[0m, time \u001b[1;36m53.22\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m140\u001b[0m: loss \u001b[1;36m2.4893\u001b[0m, time \u001b[1;36m51.06\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m150\u001b[0m: loss \u001b[1;36m2.4143\u001b[0m, time \u001b[1;36m50.87\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m160\u001b[0m: loss \u001b[1;36m2.4283\u001b[0m, time \u001b[1;36m51.99\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m170\u001b[0m: loss \u001b[1;36m2.4257\u001b[0m, time \u001b[1;36m53.09\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m180\u001b[0m: loss \u001b[1;36m2.3791\u001b[0m, time \u001b[1;36m50.44\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m190\u001b[0m: loss \u001b[1;36m2.3323\u001b[0m, time \u001b[1;36m52.81\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m200\u001b[0m: loss \u001b[1;36m2.3424\u001b[0m, time \u001b[1;36m53.69\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m210\u001b[0m: loss \u001b[1;36m2.3257\u001b[0m, time \u001b[1;36m52.34\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m220\u001b[0m: loss \u001b[1;36m2.3630\u001b[0m, time \u001b[1;36m51.08\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m230\u001b[0m: loss \u001b[1;36m2.3269\u001b[0m, time \u001b[1;36m52.13\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m240\u001b[0m: loss \u001b[1;36m2.2751\u001b[0m, time \u001b[1;36m52.81\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "step \u001b[1;36m250\u001b[0m: train loss \u001b[1;36m2.1997\u001b[0m, val loss \u001b[1;36m2.2277\u001b[0m\n",
            "saving checkpoint to \n",
            "out_test/20240120_003041_config-\u001b[1;36m500\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m384\u001b[0m-\u001b[1;36m64\u001b[0m-cuda-float16-shakespeare_char-\u001b[3;91mFalse\u001b[0m-\u001b[3;92mTrue\u001b[0m-e\n",
            "iter \u001b[1;36m250\u001b[0m: loss \u001b[1;36m2.2827\u001b[0m, time \u001b[1;36m5124.67\u001b[0m ms, mfu \u001b[1;36m1.49\u001b[0m%\n",
            "iter \u001b[1;36m260\u001b[0m: loss \u001b[1;36m2.2728\u001b[0m, time \u001b[1;36m53.66\u001b[0m ms, mfu \u001b[1;36m1.50\u001b[0m%\n",
            "iter \u001b[1;36m270\u001b[0m: loss \u001b[1;36m2.2380\u001b[0m, time \u001b[1;36m53.56\u001b[0m ms, mfu \u001b[1;36m1.51\u001b[0m%\n",
            "iter \u001b[1;36m280\u001b[0m: loss \u001b[1;36m2.2038\u001b[0m, time \u001b[1;36m52.73\u001b[0m ms, mfu \u001b[1;36m1.53\u001b[0m%\n",
            "iter \u001b[1;36m290\u001b[0m: loss \u001b[1;36m2.2445\u001b[0m, time \u001b[1;36m52.30\u001b[0m ms, mfu \u001b[1;36m1.54\u001b[0m%\n",
            "iter \u001b[1;36m300\u001b[0m: loss \u001b[1;36m2.2033\u001b[0m, time \u001b[1;36m51.67\u001b[0m ms, mfu \u001b[1;36m1.55\u001b[0m%\n",
            "iter \u001b[1;36m310\u001b[0m: loss \u001b[1;36m2.2223\u001b[0m, time \u001b[1;36m52.22\u001b[0m ms, mfu \u001b[1;36m1.56\u001b[0m%\n",
            "iter \u001b[1;36m320\u001b[0m: loss \u001b[1;36m2.1843\u001b[0m, time \u001b[1;36m52.72\u001b[0m ms, mfu \u001b[1;36m1.57\u001b[0m%\n",
            "iter \u001b[1;36m330\u001b[0m: loss \u001b[1;36m2.1800\u001b[0m, time \u001b[1;36m52.93\u001b[0m ms, mfu \u001b[1;36m1.57\u001b[0m%\n",
            "iter \u001b[1;36m340\u001b[0m: loss \u001b[1;36m2.1070\u001b[0m, time \u001b[1;36m53.20\u001b[0m ms, mfu \u001b[1;36m1.58\u001b[0m%\n",
            "iter \u001b[1;36m350\u001b[0m: loss \u001b[1;36m2.1371\u001b[0m, time \u001b[1;36m53.84\u001b[0m ms, mfu \u001b[1;36m1.58\u001b[0m%\n",
            "iter \u001b[1;36m360\u001b[0m: loss \u001b[1;36m2.1435\u001b[0m, time \u001b[1;36m53.73\u001b[0m ms, mfu \u001b[1;36m1.58\u001b[0m%\n",
            "iter \u001b[1;36m370\u001b[0m: loss \u001b[1;36m2.1212\u001b[0m, time \u001b[1;36m52.95\u001b[0m ms, mfu \u001b[1;36m1.59\u001b[0m%\n",
            "iter \u001b[1;36m380\u001b[0m: loss \u001b[1;36m2.1330\u001b[0m, time \u001b[1;36m52.78\u001b[0m ms, mfu \u001b[1;36m1.59\u001b[0m%\n",
            "iter \u001b[1;36m390\u001b[0m: loss \u001b[1;36m2.0787\u001b[0m, time \u001b[1;36m52.30\u001b[0m ms, mfu \u001b[1;36m1.60\u001b[0m%\n",
            "iter \u001b[1;36m400\u001b[0m: loss \u001b[1;36m2.1050\u001b[0m, time \u001b[1;36m51.76\u001b[0m ms, mfu \u001b[1;36m1.60\u001b[0m%\n",
            "iter \u001b[1;36m410\u001b[0m: loss \u001b[1;36m2.0801\u001b[0m, time \u001b[1;36m52.15\u001b[0m ms, mfu \u001b[1;36m1.61\u001b[0m%\n",
            "iter \u001b[1;36m420\u001b[0m: loss \u001b[1;36m2.0883\u001b[0m, time \u001b[1;36m52.74\u001b[0m ms, mfu \u001b[1;36m1.61\u001b[0m%\n",
            "iter \u001b[1;36m430\u001b[0m: loss \u001b[1;36m2.0279\u001b[0m, time \u001b[1;36m51.05\u001b[0m ms, mfu \u001b[1;36m1.62\u001b[0m%\n",
            "iter \u001b[1;36m440\u001b[0m: loss \u001b[1;36m2.0148\u001b[0m, time \u001b[1;36m52.59\u001b[0m ms, mfu \u001b[1;36m1.62\u001b[0m%\n",
            "iter \u001b[1;36m450\u001b[0m: loss \u001b[1;36m2.0287\u001b[0m, time \u001b[1;36m50.46\u001b[0m ms, mfu \u001b[1;36m1.63\u001b[0m%\n",
            "iter \u001b[1;36m460\u001b[0m: loss \u001b[1;36m2.0280\u001b[0m, time \u001b[1;36m52.65\u001b[0m ms, mfu \u001b[1;36m1.63\u001b[0m%\n",
            "iter \u001b[1;36m470\u001b[0m: loss \u001b[1;36m1.9884\u001b[0m, time \u001b[1;36m51.35\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "iter \u001b[1;36m480\u001b[0m: loss \u001b[1;36m2.0140\u001b[0m, time \u001b[1;36m51.51\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "iter \u001b[1;36m490\u001b[0m: loss \u001b[1;36m2.0285\u001b[0m, time \u001b[1;36m51.34\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "step \u001b[1;36m500\u001b[0m: train loss \u001b[1;36m1.8897\u001b[0m, val loss \u001b[1;36m1.9912\u001b[0m\n",
            "saving checkpoint to \n",
            "out_test/20240120_003041_config-\u001b[1;36m500\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m384\u001b[0m-\u001b[1;36m64\u001b[0m-cuda-float16-shakespeare_char-\u001b[3;91mFalse\u001b[0m-\u001b[3;92mTrue\u001b[0m-e\n",
            "iter \u001b[1;36m500\u001b[0m: loss \u001b[1;36m1.9881\u001b[0m, time \u001b[1;36m4877.78\u001b[0m ms, mfu \u001b[1;36m1.48\u001b[0m%\n",
            "Running command: python3 train.py --max_iters 500 --n_layer 6 --n_head 6 --n_embd 384 --block_size 64 --device cuda --dtype float16 --dataset shakespeare_char --no-use_rotary_embeddings --use_abs_pos_embeddings --tensorboard_run_name config-500-6-6-384-64-cuda-float16-shakespeare_char-False-True-s --out_dir out_test/20240120_003129_config-500-6-6-384-64-cuda-float16-shakespeare_char-False-True-s\n",
            "2024-01-20 00:31:31.701290: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-01-20 00:31:31.701345: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-01-20 00:31:31.702603: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-01-20 00:31:31.709696: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-01-20 00:31:32.686468: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "\u001b[1m{\u001b[0m\n",
            "    \u001b[32m'block_size'\u001b[0m: \u001b[1;36m64\u001b[0m,\n",
            "    \u001b[32m'n_layer'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_head'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_embd'\u001b[0m: \u001b[1;36m384\u001b[0m,\n",
            "    \u001b[32m'dropout'\u001b[0m: \u001b[1;36m0.2\u001b[0m,\n",
            "    \u001b[32m'use_post_ln'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'layernorm_variant'\u001b[0m: \u001b[32m'rmsnorm'\u001b[0m,\n",
            "    \u001b[32m'bias'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'activation_variant'\u001b[0m: \u001b[32m'gelu'\u001b[0m,\n",
            "    \u001b[32m'use_rotary_embeddings'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'rope_variant'\u001b[0m: \u001b[32m'rope'\u001b[0m,\n",
            "    \u001b[32m'shortrope_length'\u001b[0m: \u001b[1;36m16\u001b[0m,\n",
            "    \u001b[32m'use_abs_pos_embeddings'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_attn'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_output'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_beta'\u001b[0m: \u001b[1;36m0.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_gamma'\u001b[0m: \u001b[1;36m100.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_use_euler_base'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'constantmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_x_intercept'\u001b[0m: \u001b[1;36m-100.0\u001b[0m,\n",
            "    \u001b[32m'polymax_y_intercept'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n",
            "    \u001b[32m'polymax_power'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_divisor'\u001b[0m: \u001b[1;36m1000.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_use_euler_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'strongermax_strength'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'softermax_use_xmax'\u001b[0m: \u001b[3;92mTrue\u001b[0m\n",
            "\u001b[1m}\u001b[0m\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "number of parameters: 10.65M\n",
            "num decayed parameter tensors: 26, with 10,666,368 parameters\n",
            "num non-decayed parameter tensors: 13, with 4,992 parameters\n",
            "using fused AdamW: True\n",
            "step \u001b[1;36m0\u001b[0m: train loss \u001b[1;36m4.3242\u001b[0m, val loss \u001b[1;36m4.3190\u001b[0m\n",
            "iter \u001b[1;36m0\u001b[0m: loss \u001b[1;36m4.3068\u001b[0m, time \u001b[1;36m5589.33\u001b[0m ms, mfu \u001b[1;36m-100.00\u001b[0m%\n",
            "iter \u001b[1;36m10\u001b[0m: loss \u001b[1;36m3.3844\u001b[0m, time \u001b[1;36m50.32\u001b[0m ms, mfu \u001b[1;36m1.71\u001b[0m%\n",
            "iter \u001b[1;36m20\u001b[0m: loss \u001b[1;36m3.1978\u001b[0m, time \u001b[1;36m52.43\u001b[0m ms, mfu \u001b[1;36m1.71\u001b[0m%\n",
            "iter \u001b[1;36m30\u001b[0m: loss \u001b[1;36m2.9954\u001b[0m, time \u001b[1;36m51.62\u001b[0m ms, mfu \u001b[1;36m1.70\u001b[0m%\n",
            "iter \u001b[1;36m40\u001b[0m: loss \u001b[1;36m2.8807\u001b[0m, time \u001b[1;36m51.95\u001b[0m ms, mfu \u001b[1;36m1.70\u001b[0m%\n",
            "iter \u001b[1;36m50\u001b[0m: loss \u001b[1;36m2.7464\u001b[0m, time \u001b[1;36m50.23\u001b[0m ms, mfu \u001b[1;36m1.70\u001b[0m%\n",
            "iter \u001b[1;36m60\u001b[0m: loss \u001b[1;36m2.6839\u001b[0m, time \u001b[1;36m50.84\u001b[0m ms, mfu \u001b[1;36m1.70\u001b[0m%\n",
            "iter \u001b[1;36m70\u001b[0m: loss \u001b[1;36m2.6746\u001b[0m, time \u001b[1;36m51.79\u001b[0m ms, mfu \u001b[1;36m1.70\u001b[0m%\n",
            "iter \u001b[1;36m80\u001b[0m: loss \u001b[1;36m2.6093\u001b[0m, time \u001b[1;36m51.53\u001b[0m ms, mfu \u001b[1;36m1.69\u001b[0m%\n",
            "iter \u001b[1;36m90\u001b[0m: loss \u001b[1;36m2.5877\u001b[0m, time \u001b[1;36m52.78\u001b[0m ms, mfu \u001b[1;36m1.69\u001b[0m%\n",
            "iter \u001b[1;36m100\u001b[0m: loss \u001b[1;36m2.5484\u001b[0m, time \u001b[1;36m52.06\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m110\u001b[0m: loss \u001b[1;36m2.5027\u001b[0m, time \u001b[1;36m51.13\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m120\u001b[0m: loss \u001b[1;36m2.5362\u001b[0m, time \u001b[1;36m51.88\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m130\u001b[0m: loss \u001b[1;36m2.4874\u001b[0m, time \u001b[1;36m50.27\u001b[0m ms, mfu \u001b[1;36m1.69\u001b[0m%\n",
            "iter \u001b[1;36m140\u001b[0m: loss \u001b[1;36m2.4893\u001b[0m, time \u001b[1;36m52.06\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m150\u001b[0m: loss \u001b[1;36m2.4143\u001b[0m, time \u001b[1;36m52.55\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m160\u001b[0m: loss \u001b[1;36m2.4283\u001b[0m, time \u001b[1;36m51.12\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m170\u001b[0m: loss \u001b[1;36m2.4257\u001b[0m, time \u001b[1;36m52.44\u001b[0m ms, mfu \u001b[1;36m1.68\u001b[0m%\n",
            "iter \u001b[1;36m180\u001b[0m: loss \u001b[1;36m2.3791\u001b[0m, time \u001b[1;36m52.57\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m190\u001b[0m: loss \u001b[1;36m2.3323\u001b[0m, time \u001b[1;36m51.72\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m200\u001b[0m: loss \u001b[1;36m2.3424\u001b[0m, time \u001b[1;36m51.17\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m210\u001b[0m: loss \u001b[1;36m2.3257\u001b[0m, time \u001b[1;36m53.36\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m220\u001b[0m: loss \u001b[1;36m2.3630\u001b[0m, time \u001b[1;36m51.82\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m230\u001b[0m: loss \u001b[1;36m2.3269\u001b[0m, time \u001b[1;36m52.06\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m240\u001b[0m: loss \u001b[1;36m2.2751\u001b[0m, time \u001b[1;36m52.26\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "step \u001b[1;36m250\u001b[0m: train loss \u001b[1;36m2.1997\u001b[0m, val loss \u001b[1;36m2.2277\u001b[0m\n",
            "saving checkpoint to \n",
            "out_test/20240120_003129_config-\u001b[1;36m500\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m384\u001b[0m-\u001b[1;36m64\u001b[0m-cuda-float16-shakespeare_char-\u001b[3;91mFalse\u001b[0m-\u001b[3;92mTrue\u001b[0m-s\n",
            "iter \u001b[1;36m250\u001b[0m: loss \u001b[1;36m2.2827\u001b[0m, time \u001b[1;36m4748.29\u001b[0m ms, mfu \u001b[1;36m1.50\u001b[0m%\n",
            "iter \u001b[1;36m260\u001b[0m: loss \u001b[1;36m2.2728\u001b[0m, time \u001b[1;36m51.47\u001b[0m ms, mfu \u001b[1;36m1.52\u001b[0m%\n",
            "iter \u001b[1;36m270\u001b[0m: loss \u001b[1;36m2.2380\u001b[0m, time \u001b[1;36m52.87\u001b[0m ms, mfu \u001b[1;36m1.53\u001b[0m%\n",
            "iter \u001b[1;36m280\u001b[0m: loss \u001b[1;36m2.2038\u001b[0m, time \u001b[1;36m53.10\u001b[0m ms, mfu \u001b[1;36m1.54\u001b[0m%\n",
            "iter \u001b[1;36m290\u001b[0m: loss \u001b[1;36m2.2445\u001b[0m, time \u001b[1;36m52.00\u001b[0m ms, mfu \u001b[1;36m1.55\u001b[0m%\n",
            "iter \u001b[1;36m300\u001b[0m: loss \u001b[1;36m2.2033\u001b[0m, time \u001b[1;36m51.91\u001b[0m ms, mfu \u001b[1;36m1.56\u001b[0m%\n",
            "iter \u001b[1;36m310\u001b[0m: loss \u001b[1;36m2.2223\u001b[0m, time \u001b[1;36m50.80\u001b[0m ms, mfu \u001b[1;36m1.57\u001b[0m%\n",
            "iter \u001b[1;36m320\u001b[0m: loss \u001b[1;36m2.1843\u001b[0m, time \u001b[1;36m51.10\u001b[0m ms, mfu \u001b[1;36m1.59\u001b[0m%\n",
            "iter \u001b[1;36m330\u001b[0m: loss \u001b[1;36m2.1800\u001b[0m, time \u001b[1;36m51.61\u001b[0m ms, mfu \u001b[1;36m1.59\u001b[0m%\n",
            "iter \u001b[1;36m340\u001b[0m: loss \u001b[1;36m2.1070\u001b[0m, time \u001b[1;36m51.71\u001b[0m ms, mfu \u001b[1;36m1.60\u001b[0m%\n",
            "iter \u001b[1;36m350\u001b[0m: loss \u001b[1;36m2.1371\u001b[0m, time \u001b[1;36m52.02\u001b[0m ms, mfu \u001b[1;36m1.61\u001b[0m%\n",
            "iter \u001b[1;36m360\u001b[0m: loss \u001b[1;36m2.1435\u001b[0m, time \u001b[1;36m52.37\u001b[0m ms, mfu \u001b[1;36m1.61\u001b[0m%\n",
            "iter \u001b[1;36m370\u001b[0m: loss \u001b[1;36m2.1212\u001b[0m, time \u001b[1;36m51.32\u001b[0m ms, mfu \u001b[1;36m1.62\u001b[0m%\n",
            "iter \u001b[1;36m380\u001b[0m: loss \u001b[1;36m2.1330\u001b[0m, time \u001b[1;36m51.68\u001b[0m ms, mfu \u001b[1;36m1.62\u001b[0m%\n",
            "iter \u001b[1;36m390\u001b[0m: loss \u001b[1;36m2.0787\u001b[0m, time \u001b[1;36m53.15\u001b[0m ms, mfu \u001b[1;36m1.62\u001b[0m%\n",
            "iter \u001b[1;36m400\u001b[0m: loss \u001b[1;36m2.1050\u001b[0m, time \u001b[1;36m52.56\u001b[0m ms, mfu \u001b[1;36m1.62\u001b[0m%\n",
            "iter \u001b[1;36m410\u001b[0m: loss \u001b[1;36m2.0801\u001b[0m, time \u001b[1;36m51.32\u001b[0m ms, mfu \u001b[1;36m1.63\u001b[0m%\n",
            "iter \u001b[1;36m420\u001b[0m: loss \u001b[1;36m2.0883\u001b[0m, time \u001b[1;36m50.88\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "iter \u001b[1;36m430\u001b[0m: loss \u001b[1;36m2.0279\u001b[0m, time \u001b[1;36m57.04\u001b[0m ms, mfu \u001b[1;36m1.62\u001b[0m%\n",
            "iter \u001b[1;36m440\u001b[0m: loss \u001b[1;36m2.0148\u001b[0m, time \u001b[1;36m53.11\u001b[0m ms, mfu \u001b[1;36m1.62\u001b[0m%\n",
            "iter \u001b[1;36m450\u001b[0m: loss \u001b[1;36m2.0287\u001b[0m, time \u001b[1;36m53.53\u001b[0m ms, mfu \u001b[1;36m1.62\u001b[0m%\n",
            "iter \u001b[1;36m460\u001b[0m: loss \u001b[1;36m2.0280\u001b[0m, time \u001b[1;36m51.85\u001b[0m ms, mfu \u001b[1;36m1.63\u001b[0m%\n",
            "iter \u001b[1;36m470\u001b[0m: loss \u001b[1;36m1.9884\u001b[0m, time \u001b[1;36m51.28\u001b[0m ms, mfu \u001b[1;36m1.63\u001b[0m%\n",
            "iter \u001b[1;36m480\u001b[0m: loss \u001b[1;36m2.0140\u001b[0m, time \u001b[1;36m48.65\u001b[0m ms, mfu \u001b[1;36m1.65\u001b[0m%\n",
            "iter \u001b[1;36m490\u001b[0m: loss \u001b[1;36m2.0285\u001b[0m, time \u001b[1;36m52.18\u001b[0m ms, mfu \u001b[1;36m1.65\u001b[0m%\n",
            "step \u001b[1;36m500\u001b[0m: train loss \u001b[1;36m1.8897\u001b[0m, val loss \u001b[1;36m1.9912\u001b[0m\n",
            "saving checkpoint to \n",
            "out_test/20240120_003129_config-\u001b[1;36m500\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m384\u001b[0m-\u001b[1;36m64\u001b[0m-cuda-float16-shakespeare_char-\u001b[3;91mFalse\u001b[0m-\u001b[3;92mTrue\u001b[0m-s\n",
            "iter \u001b[1;36m500\u001b[0m: loss \u001b[1;36m1.9881\u001b[0m, time \u001b[1;36m5117.53\u001b[0m ms, mfu \u001b[1;36m1.48\u001b[0m%\n",
            "Running command: python3 train.py --max_iters 500 --n_layer 6 --n_head 6 --n_embd 384 --block_size 64 --device cuda --dtype float16 --dataset shakespeare_char --no-use_rotary_embeddings --use_abs_pos_embeddings --tensorboard_run_name config-500-6-6-384-64-cuda-float16-shakespeare_char-False-True-p --out_dir out_test/20240120_003218_config-500-6-6-384-64-cuda-float16-shakespeare_char-False-True-p\n",
            "2024-01-20 00:32:19.967339: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-01-20 00:32:19.967394: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-01-20 00:32:19.968723: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-01-20 00:32:19.975827: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-01-20 00:32:20.967641: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "\u001b[1m{\u001b[0m\n",
            "    \u001b[32m'block_size'\u001b[0m: \u001b[1;36m64\u001b[0m,\n",
            "    \u001b[32m'n_layer'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_head'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_embd'\u001b[0m: \u001b[1;36m384\u001b[0m,\n",
            "    \u001b[32m'dropout'\u001b[0m: \u001b[1;36m0.2\u001b[0m,\n",
            "    \u001b[32m'use_post_ln'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'layernorm_variant'\u001b[0m: \u001b[32m'rmsnorm'\u001b[0m,\n",
            "    \u001b[32m'bias'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'activation_variant'\u001b[0m: \u001b[32m'gelu'\u001b[0m,\n",
            "    \u001b[32m'use_rotary_embeddings'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'rope_variant'\u001b[0m: \u001b[32m'rope'\u001b[0m,\n",
            "    \u001b[32m'shortrope_length'\u001b[0m: \u001b[1;36m16\u001b[0m,\n",
            "    \u001b[32m'use_abs_pos_embeddings'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_attn'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_output'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_beta'\u001b[0m: \u001b[1;36m0.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_gamma'\u001b[0m: \u001b[1;36m100.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_use_euler_base'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'constantmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_x_intercept'\u001b[0m: \u001b[1;36m-100.0\u001b[0m,\n",
            "    \u001b[32m'polymax_y_intercept'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n",
            "    \u001b[32m'polymax_power'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_divisor'\u001b[0m: \u001b[1;36m1000.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_use_euler_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'strongermax_strength'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'softermax_use_xmax'\u001b[0m: \u001b[3;92mTrue\u001b[0m\n",
            "\u001b[1m}\u001b[0m\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "number of parameters: 10.65M\n",
            "num decayed parameter tensors: 26, with 10,666,368 parameters\n",
            "num non-decayed parameter tensors: 13, with 4,992 parameters\n",
            "using fused AdamW: True\n",
            "step \u001b[1;36m0\u001b[0m: train loss \u001b[1;36m4.3242\u001b[0m, val loss \u001b[1;36m4.3190\u001b[0m\n",
            "iter \u001b[1;36m0\u001b[0m: loss \u001b[1;36m4.3068\u001b[0m, time \u001b[1;36m5691.03\u001b[0m ms, mfu \u001b[1;36m-100.00\u001b[0m%\n",
            "iter \u001b[1;36m10\u001b[0m: loss \u001b[1;36m3.3844\u001b[0m, time \u001b[1;36m52.12\u001b[0m ms, mfu \u001b[1;36m1.65\u001b[0m%\n",
            "iter \u001b[1;36m20\u001b[0m: loss \u001b[1;36m3.1978\u001b[0m, time \u001b[1;36m51.07\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m30\u001b[0m: loss \u001b[1;36m2.9954\u001b[0m, time \u001b[1;36m48.47\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m40\u001b[0m: loss \u001b[1;36m2.8807\u001b[0m, time \u001b[1;36m50.93\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m50\u001b[0m: loss \u001b[1;36m2.7464\u001b[0m, time \u001b[1;36m52.69\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m60\u001b[0m: loss \u001b[1;36m2.6839\u001b[0m, time \u001b[1;36m52.11\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m70\u001b[0m: loss \u001b[1;36m2.6746\u001b[0m, time \u001b[1;36m52.23\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m80\u001b[0m: loss \u001b[1;36m2.6093\u001b[0m, time \u001b[1;36m51.10\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m90\u001b[0m: loss \u001b[1;36m2.5877\u001b[0m, time \u001b[1;36m52.58\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m100\u001b[0m: loss \u001b[1;36m2.5484\u001b[0m, time \u001b[1;36m51.14\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m110\u001b[0m: loss \u001b[1;36m2.5027\u001b[0m, time \u001b[1;36m52.38\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m120\u001b[0m: loss \u001b[1;36m2.5362\u001b[0m, time \u001b[1;36m52.91\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m130\u001b[0m: loss \u001b[1;36m2.4874\u001b[0m, time \u001b[1;36m53.45\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m140\u001b[0m: loss \u001b[1;36m2.4893\u001b[0m, time \u001b[1;36m52.35\u001b[0m ms, mfu \u001b[1;36m1.65\u001b[0m%\n",
            "iter \u001b[1;36m150\u001b[0m: loss \u001b[1;36m2.4143\u001b[0m, time \u001b[1;36m51.84\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m160\u001b[0m: loss \u001b[1;36m2.4283\u001b[0m, time \u001b[1;36m51.84\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m170\u001b[0m: loss \u001b[1;36m2.4257\u001b[0m, time \u001b[1;36m51.97\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m180\u001b[0m: loss \u001b[1;36m2.3791\u001b[0m, time \u001b[1;36m52.67\u001b[0m ms, mfu \u001b[1;36m1.65\u001b[0m%\n",
            "iter \u001b[1;36m190\u001b[0m: loss \u001b[1;36m2.3323\u001b[0m, time \u001b[1;36m53.89\u001b[0m ms, mfu \u001b[1;36m1.65\u001b[0m%\n",
            "iter \u001b[1;36m200\u001b[0m: loss \u001b[1;36m2.3424\u001b[0m, time \u001b[1;36m53.57\u001b[0m ms, mfu \u001b[1;36m1.65\u001b[0m%\n",
            "iter \u001b[1;36m210\u001b[0m: loss \u001b[1;36m2.3257\u001b[0m, time \u001b[1;36m52.79\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "iter \u001b[1;36m220\u001b[0m: loss \u001b[1;36m2.3630\u001b[0m, time \u001b[1;36m53.52\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "iter \u001b[1;36m230\u001b[0m: loss \u001b[1;36m2.3269\u001b[0m, time \u001b[1;36m54.24\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "iter \u001b[1;36m240\u001b[0m: loss \u001b[1;36m2.2751\u001b[0m, time \u001b[1;36m51.55\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "step \u001b[1;36m250\u001b[0m: train loss \u001b[1;36m2.1997\u001b[0m, val loss \u001b[1;36m2.2277\u001b[0m\n",
            "saving checkpoint to \n",
            "out_test/20240120_003218_config-\u001b[1;36m500\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m384\u001b[0m-\u001b[1;36m64\u001b[0m-cuda-float16-shakespeare_char-\u001b[3;91mFalse\u001b[0m-\u001b[3;92mTrue\u001b[0m-p\n",
            "iter \u001b[1;36m250\u001b[0m: loss \u001b[1;36m2.2827\u001b[0m, time \u001b[1;36m4939.00\u001b[0m ms, mfu \u001b[1;36m1.48\u001b[0m%\n",
            "iter \u001b[1;36m260\u001b[0m: loss \u001b[1;36m2.2728\u001b[0m, time \u001b[1;36m52.79\u001b[0m ms, mfu \u001b[1;36m1.49\u001b[0m%\n",
            "iter \u001b[1;36m270\u001b[0m: loss \u001b[1;36m2.2380\u001b[0m, time \u001b[1;36m53.62\u001b[0m ms, mfu \u001b[1;36m1.50\u001b[0m%\n",
            "iter \u001b[1;36m280\u001b[0m: loss \u001b[1;36m2.2038\u001b[0m, time \u001b[1;36m53.31\u001b[0m ms, mfu \u001b[1;36m1.52\u001b[0m%\n",
            "iter \u001b[1;36m290\u001b[0m: loss \u001b[1;36m2.2445\u001b[0m, time \u001b[1;36m53.61\u001b[0m ms, mfu \u001b[1;36m1.52\u001b[0m%\n",
            "iter \u001b[1;36m300\u001b[0m: loss \u001b[1;36m2.2033\u001b[0m, time \u001b[1;36m51.99\u001b[0m ms, mfu \u001b[1;36m1.54\u001b[0m%\n",
            "iter \u001b[1;36m310\u001b[0m: loss \u001b[1;36m2.2223\u001b[0m, time \u001b[1;36m51.85\u001b[0m ms, mfu \u001b[1;36m1.55\u001b[0m%\n",
            "iter \u001b[1;36m320\u001b[0m: loss \u001b[1;36m2.1843\u001b[0m, time \u001b[1;36m51.03\u001b[0m ms, mfu \u001b[1;36m1.56\u001b[0m%\n",
            "iter \u001b[1;36m330\u001b[0m: loss \u001b[1;36m2.1800\u001b[0m, time \u001b[1;36m51.16\u001b[0m ms, mfu \u001b[1;36m1.58\u001b[0m%\n",
            "iter \u001b[1;36m340\u001b[0m: loss \u001b[1;36m2.1070\u001b[0m, time \u001b[1;36m52.23\u001b[0m ms, mfu \u001b[1;36m1.58\u001b[0m%\n",
            "iter \u001b[1;36m350\u001b[0m: loss \u001b[1;36m2.1371\u001b[0m, time \u001b[1;36m52.32\u001b[0m ms, mfu \u001b[1;36m1.59\u001b[0m%\n",
            "iter \u001b[1;36m360\u001b[0m: loss \u001b[1;36m2.1435\u001b[0m, time \u001b[1;36m51.84\u001b[0m ms, mfu \u001b[1;36m1.60\u001b[0m%\n",
            "iter \u001b[1;36m370\u001b[0m: loss \u001b[1;36m2.1212\u001b[0m, time \u001b[1;36m51.05\u001b[0m ms, mfu \u001b[1;36m1.61\u001b[0m%\n",
            "iter \u001b[1;36m380\u001b[0m: loss \u001b[1;36m2.1330\u001b[0m, time \u001b[1;36m52.03\u001b[0m ms, mfu \u001b[1;36m1.61\u001b[0m%\n",
            "iter \u001b[1;36m390\u001b[0m: loss \u001b[1;36m2.0787\u001b[0m, time \u001b[1;36m53.71\u001b[0m ms, mfu \u001b[1;36m1.61\u001b[0m%\n",
            "iter \u001b[1;36m400\u001b[0m: loss \u001b[1;36m2.1050\u001b[0m, time \u001b[1;36m52.53\u001b[0m ms, mfu \u001b[1;36m1.61\u001b[0m%\n",
            "iter \u001b[1;36m410\u001b[0m: loss \u001b[1;36m2.0801\u001b[0m, time \u001b[1;36m53.60\u001b[0m ms, mfu \u001b[1;36m1.61\u001b[0m%\n",
            "iter \u001b[1;36m420\u001b[0m: loss \u001b[1;36m2.0883\u001b[0m, time \u001b[1;36m52.02\u001b[0m ms, mfu \u001b[1;36m1.62\u001b[0m%\n",
            "iter \u001b[1;36m430\u001b[0m: loss \u001b[1;36m2.0279\u001b[0m, time \u001b[1;36m50.44\u001b[0m ms, mfu \u001b[1;36m1.63\u001b[0m%\n",
            "iter \u001b[1;36m440\u001b[0m: loss \u001b[1;36m2.0148\u001b[0m, time \u001b[1;36m51.98\u001b[0m ms, mfu \u001b[1;36m1.63\u001b[0m%\n",
            "iter \u001b[1;36m450\u001b[0m: loss \u001b[1;36m2.0287\u001b[0m, time \u001b[1;36m52.98\u001b[0m ms, mfu \u001b[1;36m1.63\u001b[0m%\n",
            "iter \u001b[1;36m460\u001b[0m: loss \u001b[1;36m2.0280\u001b[0m, time \u001b[1;36m52.11\u001b[0m ms, mfu \u001b[1;36m1.63\u001b[0m%\n",
            "iter \u001b[1;36m470\u001b[0m: loss \u001b[1;36m1.9884\u001b[0m, time \u001b[1;36m52.31\u001b[0m ms, mfu \u001b[1;36m1.63\u001b[0m%\n",
            "iter \u001b[1;36m480\u001b[0m: loss \u001b[1;36m2.0140\u001b[0m, time \u001b[1;36m50.79\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "iter \u001b[1;36m490\u001b[0m: loss \u001b[1;36m2.0285\u001b[0m, time \u001b[1;36m51.24\u001b[0m ms, mfu \u001b[1;36m1.64\u001b[0m%\n",
            "step \u001b[1;36m500\u001b[0m: train loss \u001b[1;36m1.8897\u001b[0m, val loss \u001b[1;36m1.9912\u001b[0m\n",
            "saving checkpoint to \n",
            "out_test/20240120_003218_config-\u001b[1;36m500\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m384\u001b[0m-\u001b[1;36m64\u001b[0m-cuda-float16-shakespeare_char-\u001b[3;91mFalse\u001b[0m-\u001b[3;92mTrue\u001b[0m-p\n",
            "iter \u001b[1;36m500\u001b[0m: loss \u001b[1;36m1.9881\u001b[0m, time \u001b[1;36m4889.25\u001b[0m ms, mfu \u001b[1;36m1.48\u001b[0m%\n",
            "Running command: python3 train.py --max_iters 500 --n_layer 6 --n_head 6 --n_embd 384 --block_size 64 --device cuda --dtype float16 --dataset shakespeare_char --no-use_rotary_embeddings --use_abs_pos_embeddings --tensorboard_run_name config-500-6-6-384-64-cuda-float16-shakespeare_char-False-True-e --out_dir out_test/20240120_003306_config-500-6-6-384-64-cuda-float16-shakespeare_char-False-True-e\n",
            "2024-01-20 00:33:08.678815: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-01-20 00:33:08.678892: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-01-20 00:33:08.680734: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-01-20 00:33:08.691741: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-01-20 00:33:10.192750: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "\u001b[1m{\u001b[0m\n",
            "    \u001b[32m'block_size'\u001b[0m: \u001b[1;36m64\u001b[0m,\n",
            "    \u001b[32m'n_layer'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_head'\u001b[0m: \u001b[1;36m6\u001b[0m,\n",
            "    \u001b[32m'n_embd'\u001b[0m: \u001b[1;36m384\u001b[0m,\n",
            "    \u001b[32m'dropout'\u001b[0m: \u001b[1;36m0.2\u001b[0m,\n",
            "    \u001b[32m'use_post_ln'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'layernorm_variant'\u001b[0m: \u001b[32m'rmsnorm'\u001b[0m,\n",
            "    \u001b[32m'bias'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'activation_variant'\u001b[0m: \u001b[32m'gelu'\u001b[0m,\n",
            "    \u001b[32m'use_rotary_embeddings'\u001b[0m: \u001b[3;91mFalse\u001b[0m,\n",
            "    \u001b[32m'rope_variant'\u001b[0m: \u001b[32m'rope'\u001b[0m,\n",
            "    \u001b[32m'shortrope_length'\u001b[0m: \u001b[1;36m16\u001b[0m,\n",
            "    \u001b[32m'use_abs_pos_embeddings'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_attn'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'softmax_variant_output'\u001b[0m: \u001b[32m'softmax'\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_beta'\u001b[0m: \u001b[1;36m0.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_initial_gamma'\u001b[0m: \u001b[1;36m100.0\u001b[0m,\n",
            "    \u001b[32m'constantmax_use_euler_base'\u001b[0m: \u001b[3;92mTrue\u001b[0m,\n",
            "    \u001b[32m'constantmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_x_intercept'\u001b[0m: \u001b[1;36m-100.0\u001b[0m,\n",
            "    \u001b[32m'polymax_y_intercept'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n",
            "    \u001b[32m'polymax_power'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'polymax_divisor'\u001b[0m: \u001b[1;36m1000.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_use_euler_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'sigsoftmax_base'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'strongermax_strength'\u001b[0m: \u001b[1;36m2.0\u001b[0m,\n",
            "    \u001b[32m'softermax_use_xmax'\u001b[0m: \u001b[3;92mTrue\u001b[0m\n",
            "\u001b[1m}\u001b[0m\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "number of parameters: 10.65M\n",
            "num decayed parameter tensors: 26, with 10,666,368 parameters\n",
            "num non-decayed parameter tensors: 13, with 4,992 parameters\n",
            "using fused AdamW: True\n",
            "step \u001b[1;36m0\u001b[0m: train loss \u001b[1;36m4.3242\u001b[0m, val loss \u001b[1;36m4.3190\u001b[0m\n",
            "iter \u001b[1;36m0\u001b[0m: loss \u001b[1;36m4.3068\u001b[0m, time \u001b[1;36m5245.83\u001b[0m ms, mfu \u001b[1;36m-100.00\u001b[0m%\n",
            "iter \u001b[1;36m10\u001b[0m: loss \u001b[1;36m3.3844\u001b[0m, time \u001b[1;36m51.66\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m20\u001b[0m: loss \u001b[1;36m3.1978\u001b[0m, time \u001b[1;36m50.98\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m30\u001b[0m: loss \u001b[1;36m2.9954\u001b[0m, time \u001b[1;36m52.77\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m40\u001b[0m: loss \u001b[1;36m2.8807\u001b[0m, time \u001b[1;36m51.53\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m50\u001b[0m: loss \u001b[1;36m2.7464\u001b[0m, time \u001b[1;36m52.55\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m60\u001b[0m: loss \u001b[1;36m2.6839\u001b[0m, time \u001b[1;36m51.19\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m70\u001b[0m: loss \u001b[1;36m2.6746\u001b[0m, time \u001b[1;36m51.80\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m80\u001b[0m: loss \u001b[1;36m2.6093\u001b[0m, time \u001b[1;36m51.85\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m90\u001b[0m: loss \u001b[1;36m2.5877\u001b[0m, time \u001b[1;36m52.06\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m100\u001b[0m: loss \u001b[1;36m2.5484\u001b[0m, time \u001b[1;36m51.51\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m110\u001b[0m: loss \u001b[1;36m2.5027\u001b[0m, time \u001b[1;36m51.40\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m120\u001b[0m: loss \u001b[1;36m2.5362\u001b[0m, time \u001b[1;36m50.82\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m130\u001b[0m: loss \u001b[1;36m2.4874\u001b[0m, time \u001b[1;36m53.35\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m140\u001b[0m: loss \u001b[1;36m2.4893\u001b[0m, time \u001b[1;36m52.34\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m150\u001b[0m: loss \u001b[1;36m2.4143\u001b[0m, time \u001b[1;36m51.53\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m160\u001b[0m: loss \u001b[1;36m2.4283\u001b[0m, time \u001b[1;36m52.11\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m170\u001b[0m: loss \u001b[1;36m2.4257\u001b[0m, time \u001b[1;36m51.20\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m180\u001b[0m: loss \u001b[1;36m2.3791\u001b[0m, time \u001b[1;36m50.32\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m190\u001b[0m: loss \u001b[1;36m2.3323\u001b[0m, time \u001b[1;36m51.90\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m200\u001b[0m: loss \u001b[1;36m2.3424\u001b[0m, time \u001b[1;36m52.77\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m210\u001b[0m: loss \u001b[1;36m2.3257\u001b[0m, time \u001b[1;36m51.87\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m220\u001b[0m: loss \u001b[1;36m2.3630\u001b[0m, time \u001b[1;36m51.61\u001b[0m ms, mfu \u001b[1;36m1.67\u001b[0m%\n",
            "iter \u001b[1;36m230\u001b[0m: loss \u001b[1;36m2.3269\u001b[0m, time \u001b[1;36m52.80\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "iter \u001b[1;36m240\u001b[0m: loss \u001b[1;36m2.2751\u001b[0m, time \u001b[1;36m51.89\u001b[0m ms, mfu \u001b[1;36m1.66\u001b[0m%\n",
            "step \u001b[1;36m250\u001b[0m: train loss \u001b[1;36m2.1997\u001b[0m, val loss \u001b[1;36m2.2277\u001b[0m\n",
            "saving checkpoint to \n",
            "out_test/20240120_003306_config-\u001b[1;36m500\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m6\u001b[0m-\u001b[1;36m384\u001b[0m-\u001b[1;36m64\u001b[0m-cuda-float16-shakespeare_char-\u001b[3;91mFalse\u001b[0m-\u001b[3;92mTrue\u001b[0m-e\n",
            "iter \u001b[1;36m250\u001b[0m: loss \u001b[1;36m2.2827\u001b[0m, time \u001b[1;36m5121.39\u001b[0m ms, mfu \u001b[1;36m1.50\u001b[0m%\n",
            "iter \u001b[1;36m260\u001b[0m: loss \u001b[1;36m2.2728\u001b[0m, time \u001b[1;36m52.61\u001b[0m ms, mfu \u001b[1;36m1.51\u001b[0m%\n",
            "iter \u001b[1;36m270\u001b[0m: loss \u001b[1;36m2.2380\u001b[0m, time \u001b[1;36m52.06\u001b[0m ms, mfu \u001b[1;36m1.53\u001b[0m%\n",
            "iter \u001b[1;36m280\u001b[0m: loss \u001b[1;36m2.2038\u001b[0m, time \u001b[1;36m52.67\u001b[0m ms, mfu \u001b[1;36m1.54\u001b[0m%\n",
            "iter \u001b[1;36m290\u001b[0m: loss \u001b[1;36m2.2445\u001b[0m, time \u001b[1;36m52.52\u001b[0m ms, mfu \u001b[1;36m1.55\u001b[0m%\n",
            "iter \u001b[1;36m300\u001b[0m: loss \u001b[1;36m2.2033\u001b[0m, time \u001b[1;36m52.45\u001b[0m ms, mfu \u001b[1;36m1.56\u001b[0m%\n",
            "iter \u001b[1;36m310\u001b[0m: loss \u001b[1;36m2.2223\u001b[0m, time \u001b[1;36m52.08\u001b[0m ms, mfu \u001b[1;36m1.57\u001b[0m%\n",
            "iter \u001b[1;36m320\u001b[0m: loss \u001b[1;36m2.1843\u001b[0m, time \u001b[1;36m53.10\u001b[0m ms, mfu \u001b[1;36m1.57\u001b[0m%\n",
            "iter \u001b[1;36m330\u001b[0m: loss \u001b[1;36m2.1800\u001b[0m, time \u001b[1;36m52.99\u001b[0m ms, mfu \u001b[1;36m1.58\u001b[0m%\n",
            "iter \u001b[1;36m340\u001b[0m: loss \u001b[1;36m2.1070\u001b[0m, time \u001b[1;36m53.67\u001b[0m ms, mfu \u001b[1;36m1.58\u001b[0m%\n",
            "iter \u001b[1;36m350\u001b[0m: loss \u001b[1;36m2.1371\u001b[0m, time \u001b[1;36m53.57\u001b[0m ms, mfu \u001b[1;36m1.58\u001b[0m%\n",
            "iter \u001b[1;36m360\u001b[0m: loss \u001b[1;36m2.1435\u001b[0m, time \u001b[1;36m53.03\u001b[0m ms, mfu \u001b[1;36m1.59\u001b[0m%\n",
            "iter \u001b[1;36m370\u001b[0m: loss \u001b[1;36m2.1212\u001b[0m, time \u001b[1;36m51.70\u001b[0m ms, mfu \u001b[1;36m1.60\u001b[0m%\n",
            "iter \u001b[1;36m380\u001b[0m: loss \u001b[1;36m2.1330\u001b[0m, time \u001b[1;36m53.23\u001b[0m ms, mfu \u001b[1;36m1.60\u001b[0m%\n",
            "iter \u001b[1;36m390\u001b[0m: loss \u001b[1;36m2.0787\u001b[0m, time \u001b[1;36m52.62\u001b[0m ms, mfu \u001b[1;36m1.60\u001b[0m%\n",
            "iter \u001b[1;36m400\u001b[0m: loss \u001b[1;36m2.1050\u001b[0m, time \u001b[1;36m52.98\u001b[0m ms, mfu \u001b[1;36m1.60\u001b[0m%\n",
            "iter \u001b[1;36m410\u001b[0m: loss \u001b[1;36m2.0801\u001b[0m, time \u001b[1;36m53.33\u001b[0m ms, mfu \u001b[1;36m1.61\u001b[0m%\n",
            "iter \u001b[1;36m420\u001b[0m: loss \u001b[1;36m2.0883\u001b[0m, time \u001b[1;36m52.72\u001b[0m ms, mfu \u001b[1;36m1.61\u001b[0m%\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/lib/python3.10/subprocess.py\", line 1209, in wait\n",
            "    return self._wait(timeout=timeout)\n",
            "  File \"/usr/lib/python3.10/subprocess.py\", line 1959, in _wait\n",
            "    (pid, sts) = self._try_wait(0)\n",
            "  File \"/usr/lib/python3.10/subprocess.py\", line 1917, in _try_wait\n",
            "    (pid, sts) = os.waitpid(self.pid, wait_flags)\n",
            "KeyboardInterrupt\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/root/nanoGPT_gpu/run_experiments.py\", line 81, in <module>\n",
            "    main()\n",
            "  File \"/root/nanoGPT_gpu/run_experiments.py\", line 78, in main\n",
            "    run_command(combination, config_basename, args.output_dir, args.prefix, args.value_only, original_configurations[0])\n",
            "  File \"/root/nanoGPT_gpu/run_experiments.py\", line 66, in run_command\n",
            "    subprocess.run(base_command)\n",
            "  File \"/usr/lib/python3.10/subprocess.py\", line 505, in run\n",
            "    stdout, stderr = process.communicate(input, timeout=timeout)\n",
            "  File \"/usr/lib/python3.10/subprocess.py\", line 1146, in communicate\n",
            "    self.wait()\n",
            "  File \"/usr/lib/python3.10/subprocess.py\", line 1222, in wait\n",
            "    self._wait(timeout=sigint_timeout)\n",
            "  File \"/usr/lib/python3.10/subprocess.py\", line 1953, in _wait\n",
            "    time.sleep(delay)\n",
            "KeyboardInterrupt\n",
            "^C\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<div class=\"mrkdown-google-sans\">\n",
        "\n",
        "# **CSV Data Training**\n",
        "\n",
        "</div>"
      ],
      "metadata": {
        "id": "1DPbM_G9K7_1"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "<div class=\"mrkdown-google-sans\">\n",
        "\n",
        "## **Prepare sample data**\n",
        "\n",
        "</div>"
      ],
      "metadata": {
        "id": "NQVOG_y_K7_8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "%cd\n",
        "!rm -rf nanoGPT_gpu\n",
        "!git clone https://github.com/ReaLLMASIC/nanoGPT.git nanoGPT_gpu\n",
        "%cd nanoGPT_gpu\n",
        "!pip install --upgrade pip\n",
        "!pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118\n",
        "!pip install numpy transformers datasets tiktoken wandb tqdm tensorboard\n",
        "%cd data/csv_data\n",
        "%ls\n",
        "!bash example_sine_and_noise.sh\n",
        "!tail processed_sine_data.csv\n",
        "!python3 prepare.py -i processed_sine_data.csv"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "738acc4e-ad70-4d0b-87c7-78d9fc55a31d",
        "id": "8WYtU_cvK7_9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/root\n",
            "Cloning into 'nanoGPT_gpu'...\n",
            "remote: Enumerating objects: 1195, done.\u001b[K\n",
            "remote: Counting objects: 100% (485/485), done.\u001b[K\n",
            "remote: Compressing objects: 100% (222/222), done.\u001b[K\n",
            "remote: Total 1195 (delta 290), reused 365 (delta 255), pack-reused 710\u001b[K\n",
            "Receiving objects: 100% (1195/1195), 2.95 MiB | 29.94 MiB/s, done.\n",
            "Resolving deltas: 100% (680/680), done.\n",
            "/root/nanoGPT_gpu\n",
            "Requirement already satisfied: pip in /usr/local/lib/python3.10/dist-packages (23.1.2)\n",
            "Collecting pip\n",
            "  Downloading pip-23.3.1-py3-none-any.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m32.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: pip\n",
            "  Attempting uninstall: pip\n",
            "    Found existing installation: pip 23.1.2\n",
            "    Uninstalling pip-23.1.2:\n",
            "      Successfully uninstalled pip-23.1.2\n",
            "Successfully installed pip-23.3.1\n",
            "Looking in indexes: https://download.pytorch.org/whl/cu118\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.1.0+cu118)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (0.16.0+cu118)\n",
            "Requirement already satisfied: torchaudio in /usr/local/lib/python3.10/dist-packages (2.1.0+cu118)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.13.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch) (4.5.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.2)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2023.6.0)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch) (2.1.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from torchvision) (1.23.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from torchvision) (2.31.0)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision) (9.4.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (2.1.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision) (2023.11.17)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (1.23.5)\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.35.2)\n",
            "Collecting datasets\n",
            "  Downloading datasets-2.15.0-py3-none-any.whl.metadata (20 kB)\n",
            "Collecting tiktoken\n",
            "  Downloading tiktoken-0.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n",
            "Collecting wandb\n",
            "  Downloading wandb-0.16.0-py3-none-any.whl.metadata (9.8 kB)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (4.66.1)\n",
            "Requirement already satisfied: tensorboard in /usr/local/lib/python3.10/dist-packages (2.14.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.13.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.15.0)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.1)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (9.0.0)\n",
            "Collecting pyarrow-hotfix (from datasets)\n",
            "  Downloading pyarrow_hotfix-0.6-py3-none-any.whl.metadata (3.6 kB)\n",
            "Collecting dill<0.3.8,>=0.3.0 (from datasets)\n",
            "  Downloading dill-0.3.7-py3-none-any.whl.metadata (9.9 kB)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n",
            "Collecting multiprocess (from datasets)\n",
            "  Downloading multiprocess-0.70.15-py310-none-any.whl.metadata (7.2 kB)\n",
            "Requirement already satisfied: fsspec<=2023.10.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.1)\n",
            "Requirement already satisfied: Click!=8.0.0,>=7.1 in /usr/local/lib/python3.10/dist-packages (from wandb) (8.1.7)\n",
            "Collecting GitPython!=3.1.29,>=1.0.0 (from wandb)\n",
            "  Downloading GitPython-3.1.40-py3-none-any.whl.metadata (12 kB)\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (5.9.5)\n",
            "Collecting sentry-sdk>=1.0.0 (from wandb)\n",
            "  Downloading sentry_sdk-1.38.0-py2.py3-none-any.whl.metadata (9.7 kB)\n",
            "Collecting docker-pycreds>=0.4.0 (from wandb)\n",
            "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
            "Collecting setproctitle (from wandb)\n",
            "  Downloading setproctitle-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (9.9 kB)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb) (67.7.2)\n",
            "Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb) (1.4.4)\n",
            "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (3.20.3)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.10/dist-packages (from tensorboard) (1.4.0)\n",
            "Requirement already satisfied: grpcio>=1.48.2 in /usr/local/lib/python3.10/dist-packages (from tensorboard) (1.59.3)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.10/dist-packages (from tensorboard) (2.17.3)\n",
            "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /usr/local/lib/python3.10/dist-packages (from tensorboard) (1.0.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard) (3.5.1)\n",
            "Requirement already satisfied: six>1.9 in /usr/local/lib/python3.10/dist-packages (from tensorboard) (1.16.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard) (0.7.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard) (3.0.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.4)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.3)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Collecting gitdb<5,>=4.0.1 (from GitPython!=3.1.29,>=1.0.0->wandb)\n",
            "  Downloading gitdb-4.0.11-py3-none-any.whl.metadata (1.2 kB)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard) (5.3.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard) (0.3.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard) (4.9)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard) (1.3.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (4.5.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.11.17)\n",
            "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.10/dist-packages (from werkzeug>=1.0.1->tensorboard) (2.1.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.3.post1)\n",
            "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb)\n",
            "  Downloading smmap-5.0.1-py3-none-any.whl.metadata (4.3 kB)\n",
            "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard) (0.5.1)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard) (3.2.2)\n",
            "Downloading datasets-2.15.0-py3-none-any.whl (521 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m521.2/521.2 kB\u001b[0m \u001b[31m24.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading tiktoken-0.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.0 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m83.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading wandb-0.16.0-py3-none-any.whl (2.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m89.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading dill-0.3.7-py3-none-any.whl (115 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m11.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading GitPython-3.1.40-py3-none-any.whl (190 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m190.6/190.6 kB\u001b[0m \u001b[31m16.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading sentry_sdk-1.38.0-py2.py3-none-any.whl (252 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m252.8/252.8 kB\u001b[0m \u001b[31m19.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m11.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pyarrow_hotfix-0.6-py3-none-any.whl (7.9 kB)\n",
            "Downloading setproctitle-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\n",
            "Downloading gitdb-4.0.11-py3-none-any.whl (62 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading smmap-5.0.1-py3-none-any.whl (24 kB)\n",
            "Installing collected packages: smmap, setproctitle, sentry-sdk, pyarrow-hotfix, docker-pycreds, dill, tiktoken, multiprocess, gitdb, GitPython, wandb, datasets\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "llmx 0.0.15a0 requires cohere, which is not installed.\n",
            "llmx 0.0.15a0 requires openai, which is not installed.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed GitPython-3.1.40 datasets-2.15.0 dill-0.3.7 docker-pycreds-0.4.0 gitdb-4.0.11 multiprocess-0.70.15 pyarrow-hotfix-0.6 sentry-sdk-1.38.0 setproctitle-1.3.3 smmap-5.0.1 tiktoken-0.5.2 wandb-0.16.0\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m/root/nanoGPT_gpu/data/csv_data\n",
            "app.py           example_sine_and_noise.sh  process_csv.py           \u001b[0m\u001b[01;34mtemplates\u001b[0m/\n",
            "clean.sh         main.sh                    README.md                unprocess.py\n",
            "combine_csvs.py  prepare.py                 sine_noise_generator.py\n",
            "+ python3 sine_noise_generator.py --noise_level 0.3 --filename sine_data.csv --scientific --precision 2 --modulo 1000 --points 1000000 --split\n",
            "Time data saved to time_sine_data.csv\n",
            "Signal data saved to data_sine_data.csv\n",
            "+ set +x\n",
            "\n",
            "Preview: Generated Times\n",
            "0.00e+00\n",
            "1.00e+00\n",
            "2.00e+00\n",
            "3.00e+00\n",
            "4.00e+00\n",
            "5.00e+00\n",
            "6.00e+00\n",
            "7.00e+00\n",
            "8.00e+00\n",
            "9.00e+00\n",
            "\n",
            "\n",
            "Preview: Generated Data\n",
            "0.00e+00,-1.84e-01,-1.84e-01\n",
            "6.28e-02,7.32e-01,7.95e-01\n",
            "1.25e-01,3.07e-01,4.33e-01\n",
            "1.87e-01,-3.81e-01,-1.94e-01\n",
            "2.49e-01,3.24e-01,5.73e-01\n",
            "3.09e-01,2.73e-01,5.82e-01\n",
            "3.68e-01,-2.09e-02,3.47e-01\n",
            "4.26e-01,-2.71e-01,1.54e-01\n",
            "4.82e-01,1.12e-01,5.93e-01\n",
            "5.36e-01,-1.13e-01,4.23e-01\n",
            "\n",
            "\n",
            "\n",
            "+ python3 process_csv.py data_sine_data.csv sine_noise_sn_shuffled.csv --shuffle --exclude e\n",
            "+ set +x\n",
            "\n",
            "Preview: Shuffled Data\n",
            "c-1.84e-01a0.00e+00b-1.84e-01\n",
            "a6.28e-02b7.32e-01c7.95e-01\n",
            "c4.33e-01b3.07e-01a1.25e-01\n",
            "b-3.81e-01a1.87e-01c-1.94e-01\n",
            "b3.24e-01c5.73e-01a2.49e-01\n",
            "c5.82e-01a3.09e-01b2.73e-01\n",
            "a3.68e-01c3.47e-01b-2.09e-02\n",
            "c1.54e-01a4.26e-01b-2.71e-01\n",
            "b1.12e-01a4.82e-01c5.93e-01\n",
            "a5.36e-01c4.23e-01b-1.13e-01\n",
            "\n",
            "\n",
            "\n",
            "+ python3 combine_csvs.py -l time_sine_data.csv -r sine_noise_sn_shuffled.csv -o processed_sine_data.csv\n",
            "+ set +x\n",
            "\n",
            "Preview: Timestamps with Shuffled Data -- Use This For Training\n",
            "0.00e+00c-1.84e-01a0.00e+00b-1.84e-01\n",
            "1.00e+00a6.28e-02b7.32e-01c7.95e-01\n",
            "2.00e+00c4.33e-01b3.07e-01a1.25e-01\n",
            "3.00e+00b-3.81e-01a1.87e-01c-1.94e-01\n",
            "4.00e+00b3.24e-01c5.73e-01a2.49e-01\n",
            "5.00e+00c5.82e-01a3.09e-01b2.73e-01\n",
            "6.00e+00a3.68e-01c3.47e-01b-2.09e-02\n",
            "7.00e+00c1.54e-01a4.26e-01b-2.71e-01\n",
            "8.00e+00b1.12e-01a4.82e-01c5.93e-01\n",
            "9.00e+00a5.36e-01c4.23e-01b-1.13e-01\n",
            "+ python3 unprocess.py -i sine_noise_sn_shuffled.csv -o sine_data_unshuffled.csv -l abc -c\n",
            "+ set +x\n",
            "\n",
            "Preview: Undo shuffling Of Data\n",
            "0.00e+00,-1.84e-01,-1.84e-01\n",
            "6.28e-02,7.32e-01,7.95e-01\n",
            "1.25e-01,3.07e-01,4.33e-01\n",
            "1.87e-01,-3.81e-01,-1.94e-01\n",
            "2.49e-01,3.24e-01,5.73e-01\n",
            "3.09e-01,2.73e-01,5.82e-01\n",
            "3.68e-01,-2.09e-02,3.47e-01\n",
            "4.26e-01,-2.71e-01,1.54e-01\n",
            "4.82e-01,1.12e-01,5.93e-01\n",
            "5.36e-01,-1.13e-01,4.23e-01\n",
            "+ python3 combine_csvs.py -l time_sine_data.csv -r sine_data_unshuffled.csv -o sine_time_data_unshuffled_recombined.csv -d,\n",
            "+ set +x\n",
            "\n",
            "Preview: recombined data\n",
            "0.00e+00,0.00e+00,-1.84e-01,-1.84e-01\n",
            "1.00e+00,6.28e-02,7.32e-01,7.95e-01\n",
            "2.00e+00,1.25e-01,3.07e-01,4.33e-01\n",
            "3.00e+00,1.87e-01,-3.81e-01,-1.94e-01\n",
            "4.00e+00,2.49e-01,3.24e-01,5.73e-01\n",
            "5.00e+00,3.09e-01,2.73e-01,5.82e-01\n",
            "6.00e+00,3.68e-01,-2.09e-02,3.47e-01\n",
            "7.00e+00,4.26e-01,-2.71e-01,1.54e-01\n",
            "8.00e+00,4.82e-01,1.12e-01,5.93e-01\n",
            "9.00e+00,5.36e-01,-1.13e-01,4.23e-01\n",
            "+ results_plot_dir=./results/\n",
            "+ '[' '!' -d ./results/ ']'\n",
            "+ mkdir -p ./results/\n",
            "+ cp sine_time_data_unshuffled_recombined.csv ./results/\n",
            "+ set +x\n",
            "\n",
            "Process overview complete: use 'python3 app.py results' to view recombined graph\n",
            "9.90e+02a-5.88e-01b-2.32e-01c-8.20e-01\n",
            "9.91e+02b-3.57e-02a-5.36e-01c-5.72e-01\n",
            "9.92e+02b1.70e-01a-4.82e-01c-3.12e-01\n",
            "9.93e+02a-4.26e-01c-2.24e-01b2.02e-01\n",
            "9.94e+02b9.16e-02c-2.77e-01a-3.68e-01\n",
            "9.95e+02a-3.09e-01c-5.57e-01b-2.48e-01\n",
            "9.96e+02a-2.49e-01b-4.89e-01c-7.38e-01\n",
            "9.97e+02c2.27e-01a-1.87e-01b4.14e-01\n",
            "9.98e+02b5.28e-01c4.03e-01a-1.25e-01\n",
            "9.99e+02a-6.28e-02c1.47e-01b2.10e-01\n",
            "Length of dataset: 37,499,844\n",
            "Unique chars: \n",
            "+-.0123456789abce\n",
            "Vocab size: 18\n",
            "Train tokens: 33,749,859\n",
            "Val tokens: 3,749,985\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd\n",
        "%cd nanoGPT_gpu\n",
        "!python3 train.py --dataset=\"csv_data\" --dtype=\"float16\" --device=\"cuda\" --max_iters=2100 --block_size=1000"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4ca8e91a-b728-481f-f1d8-6940691ba03b",
        "id": "O4a_gi4AK7_-"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/root\n",
            "/root/nanoGPT_gpu\n",
            "2023-12-03 10:56:54.468821: I tensorflow/core/util/port.cc:111] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
            "2023-12-03 10:56:54.516299: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2023-12-03 10:56:54.516348: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2023-12-03 10:56:54.516391: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2023-12-03 10:56:54.524639: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2023-12-03 10:56:55.528750: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "{'block_size': 1000, 'n_layer': 6, 'n_head': 6, 'n_embd': 384, 'dropout': 0.2, 'bias': False, 'use_post_ln': False, 'use_rmsnorm': True, 'use_rotary_embeddings': True, 'rope_variant': 'rope', 'shortrope_length': 16, 'use_abs_pos_embeddings': False, 'use_softmax_variant': False, 'softmax_variant': 'softermax', 'use_softermax_xmax': False, 'strongermax_strength': 2}\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "Use GELU\n",
            "number of parameters: 10.63M\n",
            "num decayed parameter tensors: 26, with 11,007,744 parameters\n",
            "num non-decayed parameter tensors: 13, with 4,992 parameters\n",
            "using fused AdamW: True\n",
            "step 0: train loss 7.5444, val loss 7.5449\n",
            "iter 0: loss 6.7376, time 16342.83ms, mfu -100.00%\n",
            "iter 10: loss 1.9927, time 110.63ms, mfu 16.95%\n",
            "iter 20: loss 1.7774, time 110.63ms, mfu 16.95%\n",
            "iter 30: loss 1.7091, time 110.50ms, mfu 16.95%\n",
            "iter 40: loss 1.7027, time 110.55ms, mfu 16.95%\n",
            "iter 50: loss 1.6995, time 111.38ms, mfu 16.94%\n",
            "iter 60: loss 1.6833, time 111.06ms, mfu 16.94%\n",
            "iter 70: loss 1.6926, time 111.38ms, mfu 16.93%\n",
            "iter 80: loss 1.6934, time 111.20ms, mfu 16.92%\n",
            "iter 90: loss 1.6736, time 110.98ms, mfu 16.92%\n",
            "iter 100: loss 1.6692, time 110.67ms, mfu 16.92%\n",
            "iter 110: loss 1.6558, time 110.69ms, mfu 16.92%\n",
            "iter 120: loss 1.6457, time 110.56ms, mfu 16.93%\n",
            "iter 130: loss 1.6231, time 110.48ms, mfu 16.93%\n",
            "iter 140: loss 1.6614, time 110.68ms, mfu 16.93%\n",
            "iter 150: loss 1.6214, time 110.66ms, mfu 16.93%\n",
            "iter 160: loss 1.6460, time 110.53ms, mfu 16.94%\n",
            "iter 170: loss 1.5953, time 111.00ms, mfu 16.93%\n",
            "iter 180: loss 1.5865, time 111.03ms, mfu 16.93%\n",
            "iter 190: loss 1.5544, time 110.59ms, mfu 16.93%\n",
            "iter 200: loss 1.5596, time 111.33ms, mfu 16.92%\n",
            "iter 210: loss 1.5390, time 111.71ms, mfu 16.91%\n",
            "iter 220: loss 1.5313, time 110.70ms, mfu 16.91%\n",
            "iter 230: loss 1.4836, time 110.65ms, mfu 16.92%\n",
            "iter 240: loss 1.4338, time 110.66ms, mfu 16.92%\n",
            "step 250: train loss 1.3541, val loss 1.3532\n",
            "saving checkpoint to out\n",
            "iter 250: loss 1.4040, time 12845.89ms, mfu 15.24%\n",
            "iter 260: loss 1.3590, time 110.62ms, mfu 15.41%\n",
            "iter 270: loss 1.3639, time 110.99ms, mfu 15.56%\n",
            "iter 280: loss 1.3313, time 110.90ms, mfu 15.70%\n",
            "iter 290: loss 1.2576, time 111.35ms, mfu 15.81%\n",
            "iter 300: loss 1.2356, time 110.68ms, mfu 15.92%\n",
            "iter 310: loss 1.2053, time 110.58ms, mfu 16.03%\n",
            "iter 320: loss 1.1553, time 110.59ms, mfu 16.12%\n",
            "iter 330: loss 1.1678, time 110.73ms, mfu 16.20%\n",
            "iter 340: loss 1.1231, time 110.98ms, mfu 16.27%\n",
            "iter 350: loss 1.1029, time 110.61ms, mfu 16.34%\n",
            "iter 360: loss 1.0748, time 110.74ms, mfu 16.40%\n",
            "iter 370: loss 1.1038, time 110.56ms, mfu 16.46%\n",
            "iter 380: loss 1.0620, time 111.13ms, mfu 16.50%\n",
            "iter 390: loss 1.0304, time 110.74ms, mfu 16.54%\n",
            "iter 400: loss 1.0357, time 110.69ms, mfu 16.58%\n",
            "iter 410: loss 1.0335, time 110.65ms, mfu 16.62%\n",
            "iter 420: loss 1.0565, time 110.77ms, mfu 16.65%\n",
            "iter 430: loss 1.0315, time 110.64ms, mfu 16.68%\n",
            "iter 440: loss 1.0154, time 110.61ms, mfu 16.71%\n",
            "iter 450: loss 1.0023, time 111.25ms, mfu 16.72%\n",
            "iter 460: loss 0.9986, time 110.94ms, mfu 16.74%\n",
            "iter 470: loss 1.2086, time 110.91ms, mfu 16.76%\n",
            "iter 480: loss 1.0942, time 111.07ms, mfu 16.77%\n",
            "iter 490: loss 1.0150, time 111.06ms, mfu 16.78%\n",
            "step 500: train loss 0.9409, val loss 0.9413\n",
            "saving checkpoint to out\n",
            "iter 500: loss 0.9948, time 12928.16ms, mfu 15.12%\n",
            "iter 510: loss 0.9735, time 110.65ms, mfu 15.30%\n",
            "iter 520: loss 0.9655, time 110.63ms, mfu 15.47%\n",
            "iter 530: loss 0.9547, time 110.63ms, mfu 15.61%\n",
            "iter 540: loss 0.9972, time 110.66ms, mfu 15.75%\n",
            "iter 550: loss 1.0364, time 110.73ms, mfu 15.87%\n",
            "iter 560: loss 0.9946, time 110.61ms, mfu 15.97%\n",
            "iter 570: loss 0.9692, time 110.56ms, mfu 16.07%\n",
            "iter 580: loss 0.9553, time 111.09ms, mfu 16.15%\n",
            "iter 590: loss 0.9418, time 110.77ms, mfu 16.23%\n",
            "iter 600: loss 0.9421, time 110.77ms, mfu 16.30%\n",
            "iter 610: loss 0.9400, time 110.86ms, mfu 16.36%\n",
            "iter 620: loss 0.9355, time 110.72ms, mfu 16.42%\n",
            "iter 630: loss 0.9312, time 110.61ms, mfu 16.47%\n",
            "iter 640: loss 1.0033, time 110.66ms, mfu 16.52%\n",
            "iter 650: loss 1.0324, time 110.71ms, mfu 16.56%\n",
            "iter 660: loss 0.9784, time 110.69ms, mfu 16.60%\n",
            "iter 670: loss 0.9451, time 110.76ms, mfu 16.63%\n",
            "iter 680: loss 0.9243, time 110.79ms, mfu 16.66%\n",
            "iter 690: loss 0.9202, time 110.78ms, mfu 16.69%\n",
            "iter 700: loss 0.9254, time 111.56ms, mfu 16.70%\n",
            "iter 710: loss 0.9133, time 111.06ms, mfu 16.72%\n",
            "iter 720: loss 0.9058, time 110.76ms, mfu 16.74%\n",
            "iter 730: loss 0.9026, time 110.55ms, mfu 16.76%\n",
            "iter 740: loss 0.8997, time 110.59ms, mfu 16.78%\n",
            "step 750: train loss 0.8641, val loss 0.8642\n",
            "saving checkpoint to out\n",
            "iter 750: loss 0.9034, time 13112.09ms, mfu 15.12%\n",
            "iter 760: loss 0.9092, time 110.85ms, mfu 15.30%\n",
            "iter 770: loss 0.8850, time 110.66ms, mfu 15.46%\n",
            "iter 780: loss 0.8758, time 110.73ms, mfu 15.61%\n",
            "iter 790: loss 0.8628, time 110.66ms, mfu 15.74%\n",
            "iter 800: loss 0.8604, time 110.63ms, mfu 15.86%\n",
            "iter 810: loss 0.8624, time 111.33ms, mfu 15.96%\n",
            "iter 820: loss 0.8317, time 110.73ms, mfu 16.06%\n",
            "iter 830: loss 0.9286, time 110.98ms, mfu 16.14%\n",
            "iter 840: loss 0.9717, time 110.95ms, mfu 16.22%\n",
            "iter 850: loss 0.8868, time 110.65ms, mfu 16.29%\n",
            "iter 860: loss 0.8534, time 110.67ms, mfu 16.36%\n",
            "iter 870: loss 0.8303, time 110.70ms, mfu 16.42%\n",
            "iter 880: loss 0.8209, time 110.62ms, mfu 16.47%\n",
            "iter 890: loss 0.8150, time 110.70ms, mfu 16.52%\n",
            "iter 900: loss 0.7931, time 110.75ms, mfu 16.56%\n",
            "iter 910: loss 0.7895, time 110.64ms, mfu 16.60%\n",
            "iter 920: loss 0.7762, time 111.30ms, mfu 16.62%\n",
            "iter 930: loss 0.7825, time 111.73ms, mfu 16.64%\n",
            "iter 940: loss 0.7813, time 111.70ms, mfu 16.65%\n",
            "iter 950: loss 0.7611, time 110.81ms, mfu 16.68%\n",
            "iter 960: loss 0.7679, time 110.70ms, mfu 16.71%\n",
            "iter 970: loss 0.7562, time 110.66ms, mfu 16.73%\n",
            "iter 980: loss 0.7537, time 111.16ms, mfu 16.74%\n",
            "iter 990: loss 0.7391, time 110.72ms, mfu 16.76%\n",
            "step 1000: train loss 0.6850, val loss 0.6846\n",
            "saving checkpoint to out\n",
            "iter 1000: loss 0.7403, time 12937.14ms, mfu 15.10%\n",
            "iter 1010: loss 0.7621, time 110.72ms, mfu 15.29%\n",
            "iter 1020: loss 0.9037, time 110.64ms, mfu 15.45%\n",
            "iter 1030: loss 0.8074, time 110.89ms, mfu 15.60%\n",
            "iter 1040: loss 0.7615, time 111.53ms, mfu 15.72%\n",
            "iter 1050: loss 0.7344, time 110.97ms, mfu 15.84%\n",
            "iter 1060: loss 0.7204, time 110.88ms, mfu 15.94%\n",
            "iter 1070: loss 0.7034, time 110.66ms, mfu 16.05%\n",
            "iter 1080: loss 0.7063, time 110.62ms, mfu 16.14%\n",
            "iter 1090: loss 0.6935, time 110.67ms, mfu 16.22%\n",
            "iter 1100: loss 0.6845, time 110.73ms, mfu 16.29%\n",
            "iter 1110: loss 0.6772, time 110.64ms, mfu 16.35%\n",
            "iter 1120: loss 0.6756, time 110.58ms, mfu 16.41%\n",
            "iter 1130: loss 0.6711, time 110.71ms, mfu 16.47%\n",
            "iter 1140: loss 0.6592, time 111.62ms, mfu 16.50%\n",
            "iter 1150: loss 0.6550, time 111.58ms, mfu 16.53%\n",
            "iter 1160: loss 0.6603, time 110.76ms, mfu 16.57%\n",
            "iter 1170: loss 0.6517, time 110.82ms, mfu 16.61%\n",
            "iter 1180: loss 0.6375, time 110.57ms, mfu 16.64%\n",
            "iter 1190: loss 0.6305, time 110.69ms, mfu 16.67%\n",
            "iter 1200: loss 0.6339, time 110.67ms, mfu 16.70%\n",
            "iter 1210: loss 0.6291, time 110.64ms, mfu 16.72%\n",
            "iter 1220: loss 0.6237, time 111.08ms, mfu 16.74%\n",
            "iter 1230: loss 0.6186, time 110.84ms, mfu 16.76%\n",
            "iter 1240: loss 0.6076, time 110.84ms, mfu 16.77%\n",
            "step 1250: train loss 0.5561, val loss 0.5556\n",
            "saving checkpoint to out\n",
            "iter 1250: loss 0.6052, time 12947.52ms, mfu 15.11%\n",
            "iter 1260: loss 0.5977, time 111.45ms, mfu 15.28%\n",
            "iter 1270: loss 0.5958, time 110.76ms, mfu 15.45%\n",
            "iter 1280: loss 0.5910, time 110.85ms, mfu 15.59%\n",
            "iter 1290: loss 0.5817, time 111.08ms, mfu 15.72%\n",
            "iter 1300: loss 0.5872, time 110.68ms, mfu 15.85%\n",
            "iter 1310: loss 0.5829, time 110.70ms, mfu 15.95%\n",
            "iter 1320: loss 0.5715, time 110.73ms, mfu 16.05%\n",
            "iter 1330: loss 0.5721, time 111.43ms, mfu 16.13%\n",
            "iter 1340: loss 0.5613, time 110.65ms, mfu 16.21%\n",
            "iter 1350: loss 0.5662, time 110.69ms, mfu 16.29%\n",
            "iter 1360: loss 0.8652, time 110.93ms, mfu 16.35%\n",
            "iter 1370: loss 0.6581, time 111.11ms, mfu 16.40%\n",
            "iter 1380: loss 0.6032, time 110.94ms, mfu 16.45%\n",
            "iter 1390: loss 0.5810, time 110.72ms, mfu 16.50%\n",
            "iter 1400: loss 0.5595, time 110.76ms, mfu 16.54%\n",
            "iter 1410: loss 0.5602, time 110.76ms, mfu 16.58%\n",
            "iter 1420: loss 0.5529, time 110.75ms, mfu 16.62%\n",
            "iter 1430: loss 0.5531, time 110.70ms, mfu 16.65%\n",
            "iter 1440: loss 0.5484, time 110.76ms, mfu 16.68%\n",
            "iter 1450: loss 0.5445, time 111.31ms, mfu 16.69%\n",
            "iter 1460: loss 0.5404, time 110.66ms, mfu 16.72%\n",
            "iter 1470: loss 0.5417, time 110.71ms, mfu 16.74%\n",
            "iter 1480: loss 0.5373, time 111.16ms, mfu 16.75%\n",
            "iter 1490: loss 0.5355, time 110.67ms, mfu 16.77%\n",
            "step 1500: train loss 0.4950, val loss 0.4949\n",
            "saving checkpoint to out\n",
            "iter 1500: loss 0.5298, time 12955.70ms, mfu 15.11%\n",
            "iter 1510: loss 0.5286, time 111.40ms, mfu 15.28%\n",
            "iter 1520: loss 0.5261, time 111.32ms, mfu 15.44%\n",
            "iter 1530: loss 0.5291, time 110.78ms, mfu 15.59%\n",
            "iter 1540: loss 0.5232, time 110.71ms, mfu 15.72%\n",
            "iter 1550: loss 0.5358, time 111.01ms, mfu 15.84%\n",
            "iter 1560: loss 0.5373, time 110.70ms, mfu 15.95%\n",
            "iter 1570: loss 0.5278, time 111.03ms, mfu 16.04%\n",
            "iter 1580: loss 0.5218, time 110.65ms, mfu 16.13%\n",
            "iter 1590: loss 0.5180, time 110.84ms, mfu 16.21%\n",
            "iter 1600: loss 0.5190, time 110.58ms, mfu 16.29%\n",
            "iter 1610: loss 0.5202, time 110.65ms, mfu 16.35%\n",
            "iter 1620: loss 0.5136, time 110.83ms, mfu 16.41%\n",
            "iter 1630: loss 0.5118, time 111.10ms, mfu 16.46%\n",
            "iter 1640: loss 0.5088, time 110.80ms, mfu 16.50%\n",
            "iter 1650: loss 0.5056, time 110.71ms, mfu 16.55%\n",
            "iter 1660: loss 0.5111, time 110.68ms, mfu 16.59%\n",
            "iter 1670: loss 0.5025, time 110.69ms, mfu 16.62%\n",
            "iter 1680: loss 0.5089, time 111.34ms, mfu 16.64%\n",
            "iter 1690: loss 0.5037, time 110.73ms, mfu 16.67%\n",
            "iter 1700: loss 0.5014, time 111.32ms, mfu 16.69%\n",
            "iter 1710: loss 0.5038, time 110.61ms, mfu 16.72%\n",
            "iter 1720: loss 0.4953, time 110.78ms, mfu 16.74%\n",
            "iter 1730: loss 0.4941, time 110.85ms, mfu 16.76%\n",
            "iter 1740: loss 0.5019, time 110.96ms, mfu 16.77%\n",
            "step 1750: train loss 0.4958, val loss 0.4956\n",
            "iter 1750: loss 0.5778, time 12679.71ms, mfu 15.11%\n",
            "iter 1760: loss 0.6668, time 110.74ms, mfu 15.29%\n",
            "iter 1770: loss 0.5491, time 110.71ms, mfu 15.46%\n",
            "iter 1780: loss 0.5191, time 110.63ms, mfu 15.61%\n",
            "iter 1790: loss 0.4907, time 111.39ms, mfu 15.73%\n",
            "iter 1800: loss 0.4908, time 110.83ms, mfu 15.85%\n",
            "iter 1810: loss 0.4815, time 110.73ms, mfu 15.96%\n",
            "iter 1820: loss 0.4810, time 110.60ms, mfu 16.06%\n",
            "iter 1830: loss 0.4721, time 110.66ms, mfu 16.15%\n",
            "iter 1840: loss 0.4752, time 110.65ms, mfu 16.23%\n",
            "iter 1850: loss 0.4788, time 111.12ms, mfu 16.29%\n",
            "iter 1860: loss 0.4762, time 111.02ms, mfu 16.35%\n",
            "iter 1870: loss 0.4730, time 111.20ms, mfu 16.40%\n",
            "iter 1880: loss 0.4730, time 110.73ms, mfu 16.46%\n",
            "iter 1890: loss 0.4718, time 110.59ms, mfu 16.51%\n",
            "iter 1900: loss 0.4745, time 110.87ms, mfu 16.55%\n",
            "iter 1910: loss 0.4753, time 111.24ms, mfu 16.58%\n",
            "iter 1920: loss 0.4746, time 111.27ms, mfu 16.61%\n",
            "iter 1930: loss 0.4725, time 110.63ms, mfu 16.64%\n",
            "iter 1940: loss 0.4729, time 110.66ms, mfu 16.67%\n",
            "iter 1950: loss 0.4671, time 111.15ms, mfu 16.69%\n",
            "iter 1960: loss 0.4751, time 110.64ms, mfu 16.72%\n",
            "iter 1970: loss 0.4702, time 110.70ms, mfu 16.74%\n",
            "iter 1980: loss 0.4740, time 110.88ms, mfu 16.76%\n",
            "iter 1990: loss 0.4699, time 110.72ms, mfu 16.77%\n",
            "step 2000: train loss 0.4505, val loss 0.4509\n",
            "saving checkpoint to out\n",
            "iter 2000: loss 0.4702, time 12942.80ms, mfu 15.11%\n",
            "iter 2010: loss 0.4694, time 111.44ms, mfu 15.28%\n",
            "iter 2020: loss 0.4700, time 111.29ms, mfu 15.44%\n",
            "iter 2030: loss 0.4763, time 111.35ms, mfu 15.58%\n",
            "iter 2040: loss 0.4714, time 111.20ms, mfu 15.71%\n",
            "iter 2050: loss 0.4713, time 111.47ms, mfu 15.82%\n",
            "iter 2060: loss 0.4693, time 111.34ms, mfu 15.92%\n",
            "iter 2070: loss 0.4631, time 111.47ms, mfu 16.01%\n",
            "iter 2080: loss 0.4657, time 111.56ms, mfu 16.09%\n",
            "iter 2090: loss 0.4711, time 111.50ms, mfu 16.16%\n",
            "iter 2100: loss 0.4658, time 111.32ms, mfu 16.23%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd\n",
        "%cd nanoGPT_gpu\n",
        "!python3 sample.py --out_dir=\"out\" --dtype=\"float16\" --device=\"cuda\" --max_new_tokens=\"10000\" | tail -n 120 | head -n 118 | tee csv_forecast.txt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6f843223-f7ab-45d7-a2f6-e5986e1bee8e",
        "id": "0vlN49e-K8AA"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/root\n",
            "/root/nanoGPT_gpu\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd\n",
        "%cd nanoGPT_gpu\n",
        "# TODO: directly unprocess forecast that has the time and the data\n",
        "!cat csv_forecast.txt | sed 's/^........//g' > csv_data.txt\n",
        "!cat csv_forecast.txt | sed 's/\\(^........\\).*/\\1/g' > csv_time.txt\n",
        "!cat csv_data.txt\n",
        "!cat csv_time.txt\n"
      ],
      "metadata": {
        "id": "gjLCuIADK8AA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%cd\n",
        "%cd nanoGPT_gpu\n",
        "!cp csv_time.txt ./data/csv_data\n",
        "!cp csv_data.txt ./data/csv_data\n",
        "%cd ./data/csv_data\n",
        "!ls\n",
        "!python3 unprocess.py -i csv_data.txt -o csv_data_unshuffled.csv -l abc -c\n",
        "!cat csv_data_unshuffled.csv\n",
        "!python3 combine_csvs.py -l csv_time.txt -r csv_data_unshuffled.csv -o csv_extended_forecast.csv -d,\n",
        "!cat csv_extended_forecast.csv\n",
        "!python3 combine_csvs.py -l time_sine_data.csv -r data_sine_data.csv -o original.csv -d,\n",
        "!tail -n 1000 original.csv > original_last_1000.csv\n",
        "!cat original_last_1000.csv csv_extended_forecast.csv > original_plus_forecast.csv"
      ],
      "metadata": {
        "id": "Dg--_Xm6K8AB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "# Read the CSV file\n",
        "df = pd.read_csv(\"original_plus_forecast.csv\", header=None)\n",
        "# Number of rows for the x-axis\n",
        "num_rows = df.shape[0]\n",
        "\n",
        "# Create a separate plot for each column\n",
        "for i in range(df.shape[1]):\n",
        "    plt.figure(figsize=(10, 4))\n",
        "    plt.plot(range(num_rows), df[i], label=f'Column {i+1}')\n",
        "    plt.xlabel('Row Number')\n",
        "    plt.ylabel(f'Values of Column {i+1}')\n",
        "    plt.title(f'Plot of Column {i+1}')\n",
        "    plt.legend()\n",
        "    plt.show()\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "7LVI16WsK8AC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "2G2nHHjEK8AJ"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "toc_visible": true,
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}