notebook.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7fk13NtwL5-u"
      },
      "outputs": [],
      "source": [
        "!git clone https://github.com/kyegomez/LongNet.git\n",
        "\n",
        "%cd LongNet\n",
        "\n",
        "!pip install -r requirements.txt\n",
        "\n",
        "%cd test\n",
        "\n",
        "!python attention.py\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "NC6o2ezmxOIT",
        "outputId": "7d2d0d64-ab67-41c8-a88e-020385b598ff"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.0.1+cu118)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.12.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch) (4.6.3)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.11.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.2)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch) (3.25.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch) (16.0.6)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (2.1.3)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n",
            "Requirement already satisfied: einops in /usr/local/lib/python3.10/dist-packages/einops-0.7.0rc1-py3.10.egg (0.7.0rc1)\n",
            "Requirement already satisfied: torchscale in /usr/local/lib/python3.10/dist-packages (0.2.0)\n",
            "Requirement already satisfied: torch>=1.8 in /usr/local/lib/python3.10/dist-packages (from torchscale) (2.0.1+cu118)\n",
            "Requirement already satisfied: fairscale==0.4.0 in /usr/local/lib/python3.10/dist-packages (from torchscale) (0.4.0)\n",
            "Requirement already satisfied: timm==0.4.12 in /usr/local/lib/python3.10/dist-packages (from torchscale) (0.4.12)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from timm==0.4.12->torchscale) (0.15.2+cu118)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->torchscale) (3.12.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->torchscale) (4.6.3)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->torchscale) (1.11.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->torchscale) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->torchscale) (3.1.2)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->torchscale) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.8->torchscale) (3.25.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.8->torchscale) (16.0.6)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.8->torchscale) (2.1.3)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.8->torchscale) (1.3.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from torchvision->timm==0.4.12->torchscale) (1.22.4)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from torchvision->timm==0.4.12->torchscale) (2.27.1)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision->timm==0.4.12->torchscale) (8.4.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision->timm==0.4.12->torchscale) (1.26.16)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision->timm==0.4.12->torchscale) (2023.5.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision->timm==0.4.12->torchscale) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision->timm==0.4.12->torchscale) (3.4)\n",
            "Cloning into 'flash-attention'...\n",
            "remote: Enumerating objects: 2743, done.\u001b[K\n",
            "remote: Counting objects: 100% (1170/1170), done.\u001b[K\n",
            "remote: Compressing objects: 100% (200/200), done.\u001b[K\n",
            "remote: Total 2743 (delta 1016), reused 1005 (delta 968), pack-reused 1573\u001b[K\n",
            "Receiving objects: 100% (2743/2743), 3.16 MiB | 14.71 MiB/s, done.\n",
            "Resolving deltas: 100% (1817/1817), done.\n",
            "/content/flash-attention/flash-attention/flash-attention/flash-attention\n",
            "No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda'\n",
            "\n",
            "Warning: Torch did not find available GPUs on this system.\n",
            " If your intention is to cross-compile, this is not an error.\n",
            "By default, Apex will cross-compile for Pascal (compute capabilities 6.0, 6.1, 6.2),\n",
            "Volta (compute capability 7.0), Turing (compute capability 7.5),\n",
            "and, if the CUDA version is >= 11.0, Ampere (compute capability 8.0).\n",
            "If you wish to cross-compile for a single specific architecture,\n",
            "export TORCH_CUDA_ARCH_LIST=\"compute capability\" before running setup.py.\n",
            "\n",
            "\n",
            "\n",
            "torch.__version__  = 2.0.1+cu118\n",
            "\n",
            "\n",
            "Submodule 'csrc/flash_attn/cutlass' (https://github.com/NVIDIA/cutlass.git) registered for path 'csrc/flash_attn/cutlass'\n",
            "Cloning into '/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/cutlass'...\n",
            "Submodule path 'csrc/flash_attn/cutlass': checked out '319a389f42b776fae5701afcb943fc03be5b5c25'\n",
            "running install\n",
            "/usr/local/lib/python3.10/dist-packages/setuptools/_distutils/cmd.py:66: SetuptoolsDeprecationWarning: setup.py install is deprecated.\n",
            "!!\n",
            "\n",
            "        ********************************************************************************\n",
            "        Please avoid running ``setup.py`` directly.\n",
            "        Instead, use pypa/build, pypa/installer, pypa/build or\n",
            "        other standards-based tools.\n",
            "\n",
            "        See https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html for details.\n",
            "        ********************************************************************************\n",
            "\n",
            "!!\n",
            "  self.initialize_options()\n",
            "/usr/local/lib/python3.10/dist-packages/setuptools/_distutils/cmd.py:66: EasyInstallDeprecationWarning: easy_install command is deprecated.\n",
            "!!\n",
            "\n",
            "        ********************************************************************************\n",
            "        Please avoid running ``setup.py`` and ``easy_install``.\n",
            "        Instead, use pypa/build, pypa/installer, pypa/build or\n",
            "        other standards-based tools.\n",
            "\n",
            "        See https://github.com/pypa/setuptools/issues/917 for details.\n",
            "        ********************************************************************************\n",
            "\n",
            "!!\n",
            "  self.initialize_options()\n",
            "running bdist_egg\n",
            "running egg_info\n",
            "creating flash_attn.egg-info\n",
            "writing flash_attn.egg-info/PKG-INFO\n",
            "writing dependency_links to flash_attn.egg-info/dependency_links.txt\n",
            "writing requirements to flash_attn.egg-info/requires.txt\n",
            "writing top-level names to flash_attn.egg-info/top_level.txt\n",
            "writing manifest file 'flash_attn.egg-info/SOURCES.txt'\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/cpp_extension.py:476: UserWarning: Attempted to use ninja as the BuildExtension backend but we could not find ninja.. Falling back to using the slow distutils backend.\n",
            "  warnings.warn(msg.format('we could not find ninja.'))\n",
            "reading manifest file 'flash_attn.egg-info/SOURCES.txt'\n",
            "reading manifest template 'MANIFEST.in'\n",
            "warning: no files found matching '*.cu' under directory 'flash_attn'\n",
            "warning: no files found matching '*.h' under directory 'flash_attn'\n",
            "warning: no files found matching '*.cuh' under directory 'flash_attn'\n",
            "warning: no files found matching '*.cpp' under directory 'flash_attn'\n",
            "adding license file 'LICENSE'\n",
            "adding license file 'AUTHORS'\n",
            "writing manifest file 'flash_attn.egg-info/SOURCES.txt'\n",
            "installing library code to build/bdist.linux-x86_64/egg\n",
            "running install_lib\n",
            "running build_py\n",
            "creating build\n",
            "creating build/lib.linux-x86_64-cpython-310\n",
            "creating build/lib.linux-x86_64-cpython-310/flash_attn\n",
            "copying flash_attn/bert_padding.py -> build/lib.linux-x86_64-cpython-310/flash_attn\n",
            "copying flash_attn/flash_blocksparse_attention.py -> build/lib.linux-x86_64-cpython-310/flash_attn\n",
            "copying flash_attn/flash_attn_interface.py -> build/lib.linux-x86_64-cpython-310/flash_attn\n",
            "copying flash_attn/flash_blocksparse_attn_interface.py -> build/lib.linux-x86_64-cpython-310/flash_attn\n",
            "copying flash_attn/flash_attn_triton_og.py -> build/lib.linux-x86_64-cpython-310/flash_attn\n",
            "copying flash_attn/flash_attention.py -> build/lib.linux-x86_64-cpython-310/flash_attn\n",
            "copying flash_attn/__init__.py -> build/lib.linux-x86_64-cpython-310/flash_attn\n",
            "copying flash_attn/flash_attn_triton.py -> build/lib.linux-x86_64-cpython-310/flash_attn\n",
            "copying flash_attn/fused_softmax.py -> build/lib.linux-x86_64-cpython-310/flash_attn\n",
            "creating build/lib.linux-x86_64-cpython-310/flash_attn/losses\n",
            "copying flash_attn/losses/__init__.py -> build/lib.linux-x86_64-cpython-310/flash_attn/losses\n",
            "copying flash_attn/losses/cross_entropy.py -> build/lib.linux-x86_64-cpython-310/flash_attn/losses\n",
            "creating build/lib.linux-x86_64-cpython-310/flash_attn/layers\n",
            "copying flash_attn/layers/__init__.py -> build/lib.linux-x86_64-cpython-310/flash_attn/layers\n",
            "copying flash_attn/layers/rotary.py -> build/lib.linux-x86_64-cpython-310/flash_attn/layers\n",
            "copying flash_attn/layers/patch_embed.py -> build/lib.linux-x86_64-cpython-310/flash_attn/layers\n",
            "creating build/lib.linux-x86_64-cpython-310/flash_attn/utils\n",
            "copying flash_attn/utils/distributed.py -> build/lib.linux-x86_64-cpython-310/flash_attn/utils\n",
            "copying flash_attn/utils/__init__.py -> build/lib.linux-x86_64-cpython-310/flash_attn/utils\n",
            "copying flash_attn/utils/generation.py -> build/lib.linux-x86_64-cpython-310/flash_attn/utils\n",
            "copying flash_attn/utils/benchmark.py -> build/lib.linux-x86_64-cpython-310/flash_attn/utils\n",
            "copying flash_attn/utils/pretrained.py -> build/lib.linux-x86_64-cpython-310/flash_attn/utils\n",
            "creating build/lib.linux-x86_64-cpython-310/flash_attn/ops\n",
            "copying flash_attn/ops/activations.py -> build/lib.linux-x86_64-cpython-310/flash_attn/ops\n",
            "copying flash_attn/ops/fused_dense.py -> build/lib.linux-x86_64-cpython-310/flash_attn/ops\n",
            "copying flash_attn/ops/__init__.py -> build/lib.linux-x86_64-cpython-310/flash_attn/ops\n",
            "copying flash_attn/ops/rms_norm.py -> build/lib.linux-x86_64-cpython-310/flash_attn/ops\n",
            "copying flash_attn/ops/layer_norm.py -> build/lib.linux-x86_64-cpython-310/flash_attn/ops\n",
            "creating build/lib.linux-x86_64-cpython-310/flash_attn/models\n",
            "copying flash_attn/models/gptj.py -> build/lib.linux-x86_64-cpython-310/flash_attn/models\n",
            "copying flash_attn/models/vit.py -> build/lib.linux-x86_64-cpython-310/flash_attn/models\n",
            "copying flash_attn/models/bert.py -> build/lib.linux-x86_64-cpython-310/flash_attn/models\n",
            "copying flash_attn/models/gpt.py -> build/lib.linux-x86_64-cpython-310/flash_attn/models\n",
            "copying flash_attn/models/llama.py -> build/lib.linux-x86_64-cpython-310/flash_attn/models\n",
            "copying flash_attn/models/__init__.py -> build/lib.linux-x86_64-cpython-310/flash_attn/models\n",
            "copying flash_attn/models/opt.py -> build/lib.linux-x86_64-cpython-310/flash_attn/models\n",
            "copying flash_attn/models/gpt_neox.py -> build/lib.linux-x86_64-cpython-310/flash_attn/models\n",
            "creating build/lib.linux-x86_64-cpython-310/flash_attn/modules\n",
            "copying flash_attn/modules/mha.py -> build/lib.linux-x86_64-cpython-310/flash_attn/modules\n",
            "copying flash_attn/modules/block.py -> build/lib.linux-x86_64-cpython-310/flash_attn/modules\n",
            "copying flash_attn/modules/mlp.py -> build/lib.linux-x86_64-cpython-310/flash_attn/modules\n",
            "copying flash_attn/modules/__init__.py -> build/lib.linux-x86_64-cpython-310/flash_attn/modules\n",
            "copying flash_attn/modules/embedding.py -> build/lib.linux-x86_64-cpython-310/flash_attn/modules\n",
            "running build_ext\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/cpp_extension.py:398: UserWarning: There are no x86_64-linux-gnu-g++ version bounds defined for CUDA version 11.8\n",
            "  warnings.warn(f'There are no {compiler_name} version bounds defined for CUDA version {cuda_str_version}')\n",
            "building 'flash_attn_cuda' extension\n",
            "creating build/temp.linux-x86_64-cpython-310\n",
            "creating build/temp.linux-x86_64-cpython-310/csrc\n",
            "creating build/temp.linux-x86_64-cpython-310/csrc/flash_attn\n",
            "creating build/temp.linux-x86_64-cpython-310/csrc/flash_attn/src\n",
            "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -fPIC -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/src -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/cutlass/include -I/usr/local/lib/python3.10/dist-packages/torch/include -I/usr/local/lib/python3.10/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.10/dist-packages/torch/include/TH -I/usr/local/lib/python3.10/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.10 -c csrc/flash_attn/fmha_api.cpp -o build/temp.linux-x86_64-cpython-310/csrc/flash_attn/fmha_api.o -O3 -std=c++17 -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\\\"_gcc\\\" -DPYBIND11_STDLIB=\\\"_libstdcpp\\\" -DPYBIND11_BUILD_ABI=\\\"_cxxabi1011\\\" -DTORCH_EXTENSION_NAME=flash_attn_cuda -D_GLIBCXX_USE_CXX11_ABI=0\n",
            "In file included from \u001b[01m\u001b[K/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/src/fmha.h:42\u001b[m\u001b[K,\n",
            "                 from \u001b[01m\u001b[Kcsrc/flash_attn/fmha_api.cpp:33\u001b[m\u001b[K:\n",
            "\u001b[01m\u001b[K/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/src/fmha_utils.h:\u001b[m\u001b[K In function ‘\u001b[01m\u001b[Kvoid set_alpha(uint32_t&, float, Data_type)\u001b[m\u001b[K’:\n",
            "\u001b[01m\u001b[K/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/src/fmha_utils.h:63:53:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kdereferencing type-punned pointer will break strict-aliasing rules [\u001b[01;35m\u001b[K-Wstrict-aliasing\u001b[m\u001b[K]\n",
            "   63 |         alpha = reinterpret_cast<const uint32_t &>( \u001b[01;35m\u001b[Kh2\u001b[m\u001b[K );\n",
            "      |                                                     \u001b[01;35m\u001b[K^~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[K/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/src/fmha_utils.h:68:53:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kdereferencing type-punned pointer will break strict-aliasing rules [\u001b[01;35m\u001b[K-Wstrict-aliasing\u001b[m\u001b[K]\n",
            "   68 |         alpha = reinterpret_cast<const uint32_t &>( \u001b[01;35m\u001b[Kh2\u001b[m\u001b[K );\n",
            "      |                                                     \u001b[01;35m\u001b[K^~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[K/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/src/fmha_utils.h:70:53:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kdereferencing type-punned pointer will break strict-aliasing rules [\u001b[01;35m\u001b[K-Wstrict-aliasing\u001b[m\u001b[K]\n",
            "   70 |         alpha = reinterpret_cast<const uint32_t &>( \u001b[01;35m\u001b[Knorm\u001b[m\u001b[K );\n",
            "      |                                                     \u001b[01;35m\u001b[K^~~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[Kcsrc/flash_attn/fmha_api.cpp:\u001b[m\u001b[K In function ‘\u001b[01m\u001b[Kvoid set_params_fprop(FMHA_fprop_params&, size_t, size_t, size_t, size_t, size_t, at::Tensor, at::Tensor, at::Tensor, at::Tensor, void*, void*, void*, void*, void*, float, float, bool, int)\u001b[m\u001b[K’:\n",
            "\u001b[01m\u001b[Kcsrc/flash_attn/fmha_api.cpp:64:38:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kvoid* memset(void*, int, size_t)\u001b[m\u001b[K’ clearing an object of non-trivial type ‘\u001b[01m\u001b[Kstruct FMHA_fprop_params\u001b[m\u001b[K’; use assignment or value-initialization instead [\u001b[01;35m\u001b[K-Wclass-memaccess\u001b[m\u001b[K]\n",
            "   64 |     memset(&params, 0, sizeof(params)\u001b[01;35m\u001b[K)\u001b[m\u001b[K;\n",
            "      |                                      \u001b[01;35m\u001b[K^\u001b[m\u001b[K\n",
            "In file included from \u001b[01m\u001b[Kcsrc/flash_attn/fmha_api.cpp:33\u001b[m\u001b[K:\n",
            "\u001b[01m\u001b[K/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/src/fmha.h:75:8:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[K‘\u001b[01m\u001b[Kstruct FMHA_fprop_params\u001b[m\u001b[K’ declared here\n",
            "   75 | struct \u001b[01;36m\u001b[KFMHA_fprop_params\u001b[m\u001b[K : public Qkv_params {\n",
            "      |        \u001b[01;36m\u001b[K^~~~~~~~~~~~~~~~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[Kcsrc/flash_attn/fmha_api.cpp:60:15:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kunused variable ‘\u001b[01m\u001b[Kacc_type\u001b[m\u001b[K’ [\u001b[01;35m\u001b[K-Wunused-variable\u001b[m\u001b[K]\n",
            "   60 |     Data_type \u001b[01;35m\u001b[Kacc_type\u001b[m\u001b[K = DATA_TYPE_FP32;\n",
            "      |               \u001b[01;35m\u001b[K^~~~~~~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[Kcsrc/flash_attn/fmha_api.cpp:\u001b[m\u001b[K In function ‘\u001b[01m\u001b[Kstd::vector<at::Tensor> mha_fwd(const at::Tensor&, const at::Tensor&, const at::Tensor&, at::Tensor&, const at::Tensor&, const at::Tensor&, int, int, float, float, bool, bool, bool, int, c10::optional<at::Generator>)\u001b[m\u001b[K’:\n",
            "\u001b[01m\u001b[Kcsrc/flash_attn/fmha_api.cpp:208:10:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kunused variable ‘\u001b[01m\u001b[Kis_sm80\u001b[m\u001b[K’ [\u001b[01;35m\u001b[K-Wunused-variable\u001b[m\u001b[K]\n",
            "  208 |     bool \u001b[01;35m\u001b[Kis_sm80\u001b[m\u001b[K = dprops->major == 8 && dprops->minor == 0;\n",
            "      |          \u001b[01;35m\u001b[K^~~~~~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[Kcsrc/flash_attn/fmha_api.cpp:\u001b[m\u001b[K In function ‘\u001b[01m\u001b[Kstd::vector<at::Tensor> mha_fwd_block(const at::Tensor&, const at::Tensor&, const at::Tensor&, const at::Tensor&, const at::Tensor&, const at::Tensor&, int, int, float, float, bool, bool, c10::optional<at::Generator>)\u001b[m\u001b[K’:\n",
            "\u001b[01m\u001b[Kcsrc/flash_attn/fmha_api.cpp:533:10:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kunused variable ‘\u001b[01m\u001b[Kis_sm80\u001b[m\u001b[K’ [\u001b[01;35m\u001b[K-Wunused-variable\u001b[m\u001b[K]\n",
            "  533 |     bool \u001b[01;35m\u001b[Kis_sm80\u001b[m\u001b[K = dprops->major == 8 && dprops->minor == 0;\n",
            "      |          \u001b[01;35m\u001b[K^~~~~~~\u001b[m\u001b[K\n",
            "/usr/local/cuda/bin/nvcc -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/src -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/cutlass/include -I/usr/local/lib/python3.10/dist-packages/torch/include -I/usr/local/lib/python3.10/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.10/dist-packages/torch/include/TH -I/usr/local/lib/python3.10/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.10 -c csrc/flash_attn/src/fmha_block_dgrad_fp16_kernel_loop.sm80.cu -o build/temp.linux-x86_64-cpython-310/csrc/flash_attn/src/fmha_block_dgrad_fp16_kernel_loop.sm80.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -O3 -std=c++17 -U__CUDA_NO_HALF_OPERATORS__ -U__CUDA_NO_HALF_CONVERSIONS__ -U__CUDA_NO_HALF2_OPERATORS__ -U__CUDA_NO_BFLOAT16_CONVERSIONS__ --expt-relaxed-constexpr --expt-extended-lambda --use_fast_math --ptxas-options=-v -lineinfo -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_90,code=sm_90 --threads 4 -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\\\"_gcc\\\" -DPYBIND11_STDLIB=\\\"_libstdcpp\\\" -DPYBIND11_BUILD_ABI=\\\"_cxxabi1011\\\" -DTORCH_EXTENSION_NAME=flash_attn_cuda -D_GLIBCXX_USE_CXX11_ABI=0\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "ptxas info    : 25 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 251 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 239 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 239 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 241 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 241 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 254 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 242 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 246 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 240 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 238 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 209 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 210 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 230 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 225 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 248 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 244 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 194 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 206 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 208 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 218 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 164 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 165 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 196 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 197 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 206 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 211 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 233 registers\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 233 registers\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "ptxas info    : 25 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 16 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 254 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 254 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 4 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 242 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 254 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 248 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 223 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 223 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 241 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 240 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 254 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 250 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 245 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 246 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 194 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 206 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 218 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 225 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 176 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 175 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 199 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 206 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 210 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 212 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 232 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 240 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : 25 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 253 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 254 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 12 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 20 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 249 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 245 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 249 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 231 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 231 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 242 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 240 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 245 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 251 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 245 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 254 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 203 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 215 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 212 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 226 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 186 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 187 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 200 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 202 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 216 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 221 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 234 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z47fmha_block_dgrad_fp16_sm80_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 242 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "/usr/local/cuda/bin/nvcc -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/src -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/cutlass/include -I/usr/local/lib/python3.10/dist-packages/torch/include -I/usr/local/lib/python3.10/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.10/dist-packages/torch/include/TH -I/usr/local/lib/python3.10/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.10 -c csrc/flash_attn/src/fmha_block_fprop_fp16_kernel.sm80.cu -o build/temp.linux-x86_64-cpython-310/csrc/flash_attn/src/fmha_block_fprop_fp16_kernel.sm80.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -O3 -std=c++17 -U__CUDA_NO_HALF_OPERATORS__ -U__CUDA_NO_HALF_CONVERSIONS__ -U__CUDA_NO_HALF2_OPERATORS__ -U__CUDA_NO_BFLOAT16_CONVERSIONS__ --expt-relaxed-constexpr --expt-extended-lambda --use_fast_math --ptxas-options=-v -lineinfo -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_90,code=sm_90 --threads 4 -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\\\"_gcc\\\" -DPYBIND11_STDLIB=\\\"_libstdcpp\\\" -DPYBIND11_BUILD_ABI=\\\"_cxxabi1011\\\" -DTORCH_EXTENSION_NAME=flash_attn_cuda -D_GLIBCXX_USE_CXX11_ABI=0\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "ptxas info    : 25 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    40 bytes stack frame, 56 bytes spill stores, 64 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    32 bytes stack frame, 40 bytes spill stores, 40 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    16 bytes stack frame, 20 bytes spill stores, 20 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    16 bytes stack frame, 20 bytes spill stores, 20 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    16 bytes stack frame, 20 bytes spill stores, 20 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    24 bytes stack frame, 20 bytes spill stores, 20 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 171 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 190 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 187 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 202 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 224 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 232 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 227 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 235 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 156 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 201 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 199 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 218 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 212 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 241 registers\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 228 registers\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "ptxas info    : 25 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    32 bytes stack frame, 32 bytes spill stores, 36 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 4 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    24 bytes stack frame, 48 bytes spill stores, 48 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 4 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 4 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 176 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 190 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 188 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 203 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 222 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 232 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 218 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 236 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 180 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 186 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 200 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 207 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 212 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 224 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 223 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : 25 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    16 bytes stack frame, 16 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 180 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 198 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 198 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 209 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 220 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 224 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 221 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 228 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 173 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 192 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 194 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 206 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 204 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 220 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z38fmha_block_fprop_fp16_sm80_loop_kernelI18FMHA_kernel_traitsILi256ELi16ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 225 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "/usr/local/cuda/bin/nvcc -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/src -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/cutlass/include -I/usr/local/lib/python3.10/dist-packages/torch/include -I/usr/local/lib/python3.10/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.10/dist-packages/torch/include/TH -I/usr/local/lib/python3.10/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.10 -c csrc/flash_attn/src/fmha_bwd_hdim128.cu -o build/temp.linux-x86_64-cpython-310/csrc/flash_attn/src/fmha_bwd_hdim128.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -O3 -std=c++17 -U__CUDA_NO_HALF_OPERATORS__ -U__CUDA_NO_HALF_CONVERSIONS__ -U__CUDA_NO_HALF2_OPERATORS__ -U__CUDA_NO_BFLOAT16_CONVERSIONS__ --expt-relaxed-constexpr --expt-extended-lambda --use_fast_math --ptxas-options=-v -lineinfo -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_90,code=sm_90 --threads 4 -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\\\"_gcc\\\" -DPYBIND11_STDLIB=\\\"_libstdcpp\\\" -DPYBIND11_BUILD_ABI=\\\"_cxxabi1011\\\" -DTORCH_EXTENSION_NAME=flash_attn_cuda -D_GLIBCXX_USE_CXX11_ABI=0\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "csrc/flash_attn/src/fmha_bwd_launch_template.h(64): warning #177-D: variable \"M\" was declared but never referenced\n",
            "\n",
            "csrc/flash_attn/src/fmha_bwd_launch_template.h(64): warning #177-D: variable \"M\" was declared but never referenced\n",
            "\n",
            "ptxas info    : 9 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 248 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 250 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 251 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 253 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 4 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfEEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfEEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 26 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 250 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 249 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 251 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 248 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 250 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    32 bytes stack frame, 36 bytes spill stores, 36 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 248 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 250 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 251 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 253 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 4 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 25 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 250 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 249 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 251 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 248 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 250 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    32 bytes stack frame, 36 bytes spill stores, 36 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "csrc/flash_attn/src/fmha_bwd_launch_template.h(64): warning #177-D: variable \"M\" was declared but never referenced\n",
            "\n",
            "csrc/flash_attn/src/fmha_bwd_launch_template.h(64): warning #177-D: variable \"M\" was declared but never referenced\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "csrc/flash_attn/src/fmha_bwd_launch_template.h(64): warning #177-D: variable \"M\" was declared but never referenced\n",
            "\n",
            "csrc/flash_attn/src/fmha_bwd_launch_template.h(64): warning #177-D: variable \"M\" was declared but never referenced\n",
            "\n",
            "ptxas info    : 9 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 249 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 250 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 251 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    32 bytes stack frame, 40 bytes spill stores, 44 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfEEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfEEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 25 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 251 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 251 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 249 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 250 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    40 bytes stack frame, 48 bytes spill stores, 52 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 249 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 250 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 251 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    32 bytes stack frame, 40 bytes spill stores, 44 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 23 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 251 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 251 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 249 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 250 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    40 bytes stack frame, 48 bytes spill stores, 52 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : 7246 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 250 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 250 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 4 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    32 bytes stack frame, 32 bytes spill stores, 36 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfEEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfEEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 32 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 254 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 254 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 24 bytes spill stores, 44 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    24 bytes stack frame, 24 bytes spill stores, 24 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 20 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 4 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    24 bytes stack frame, 44 bytes spill stores, 56 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    64 bytes stack frame, 80 bytes spill stores, 100 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 12 bytes spill stores, 24 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 4 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    72 bytes stack frame, 92 bytes spill stores, 112 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 38 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 16 bytes spill stores, 32 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 40 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    88 bytes stack frame, 116 bytes spill stores, 156 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "/usr/local/cuda/bin/nvcc -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/src -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/cutlass/include -I/usr/local/lib/python3.10/dist-packages/torch/include -I/usr/local/lib/python3.10/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.10/dist-packages/torch/include/TH -I/usr/local/lib/python3.10/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.10 -c csrc/flash_attn/src/fmha_bwd_hdim32.cu -o build/temp.linux-x86_64-cpython-310/csrc/flash_attn/src/fmha_bwd_hdim32.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -O3 -std=c++17 -U__CUDA_NO_HALF_OPERATORS__ -U__CUDA_NO_HALF_CONVERSIONS__ -U__CUDA_NO_HALF2_OPERATORS__ -U__CUDA_NO_BFLOAT16_CONVERSIONS__ --expt-relaxed-constexpr --expt-extended-lambda --use_fast_math --ptxas-options=-v -lineinfo -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_90,code=sm_90 --threads 4 -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\\\"_gcc\\\" -DPYBIND11_STDLIB=\\\"_libstdcpp\\\" -DPYBIND11_BUILD_ABI=\\\"_cxxabi1011\\\" -DTORCH_EXTENSION_NAME=flash_attn_cuda -D_GLIBCXX_USE_CXX11_ABI=0\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "csrc/flash_attn/src/fmha_bwd_launch_template.h(64): warning #177-D: variable \"M\" was declared but never referenced\n",
            "\n",
            "csrc/flash_attn/src/fmha_bwd_launch_template.h(64): warning #177-D: variable \"M\" was declared but never referenced\n",
            "\n",
            "ptxas info    : 17 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 217 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 215 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 228 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 233 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 226 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 233 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 233 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 244 registers\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 26 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 225 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 226 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 224 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 241 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 225 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 224 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 236 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 236 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 158 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 156 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 164 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 178 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 158 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 179 registers\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 26 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 162 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 152 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 196 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 203 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 200 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 205 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 217 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 215 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 228 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 233 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 226 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 233 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 233 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 244 registers\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 25 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 225 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 226 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 224 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 241 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 225 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 224 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 236 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 236 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 158 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 156 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 164 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 178 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 158 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 179 registers\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 25 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 162 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 152 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 196 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 203 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 200 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 205 registers\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "csrc/flash_attn/src/fmha_bwd_launch_template.h(64): warning #177-D: variable \"M\" was declared but never referenced\n",
            "\n",
            "csrc/flash_attn/src/fmha_bwd_launch_template.h(64): warning #177-D: variable \"M\" was declared but never referenced\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "csrc/flash_attn/src/fmha_bwd_launch_template.h(64): warning #177-D: variable \"M\" was declared but never referenced\n",
            "\n",
            "csrc/flash_attn/src/fmha_bwd_launch_template.h(64): warning #177-D: variable \"M\" was declared but never referenced\n",
            "\n",
            "ptxas info    : 17 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 218 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 215 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 231 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 235 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 228 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 230 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 233 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 245 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 25 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 224 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 219 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 226 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 238 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 223 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 224 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 241 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 244 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 162 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 164 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 171 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 182 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 164 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 170 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 179 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 185 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 25 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 164 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 182 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 187 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 186 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 202 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 218 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 215 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 231 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 235 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 228 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 230 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 233 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 245 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 23 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 224 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 219 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 226 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 238 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 223 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 224 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 241 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 244 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 162 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 164 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 171 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 182 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 164 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 170 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 179 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 185 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 23 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 164 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 182 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 187 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 186 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 202 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : 14303 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 219 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 215 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 231 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 237 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 227 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 228 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 235 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 241 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 32 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 226 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 224 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 225 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 239 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 221 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 226 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 235 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 242 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 164 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 172 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 186 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 163 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 178 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 188 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 32 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 158 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 181 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 190 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 164 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 184 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 206 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 219 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 215 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 233 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 243 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 229 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 230 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 237 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 251 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 36 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 226 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 224 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 235 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 245 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 236 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 238 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 239 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 250 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 164 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 174 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 186 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 174 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 180 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 190 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 36 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 165 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 182 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 190 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 179 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 186 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 206 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "/usr/local/cuda/bin/nvcc -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/src -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/cutlass/include -I/usr/local/lib/python3.10/dist-packages/torch/include -I/usr/local/lib/python3.10/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.10/dist-packages/torch/include/TH -I/usr/local/lib/python3.10/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.10 -c csrc/flash_attn/src/fmha_bwd_hdim64.cu -o build/temp.linux-x86_64-cpython-310/csrc/flash_attn/src/fmha_bwd_hdim64.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -O3 -std=c++17 -U__CUDA_NO_HALF_OPERATORS__ -U__CUDA_NO_HALF_CONVERSIONS__ -U__CUDA_NO_HALF2_OPERATORS__ -U__CUDA_NO_BFLOAT16_CONVERSIONS__ --expt-relaxed-constexpr --expt-extended-lambda --use_fast_math --ptxas-options=-v -lineinfo -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_90,code=sm_90 --threads 4 -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\\\"_gcc\\\" -DPYBIND11_STDLIB=\\\"_libstdcpp\\\" -DPYBIND11_BUILD_ABI=\\\"_cxxabi1011\\\" -DTORCH_EXTENSION_NAME=flash_attn_cuda -D_GLIBCXX_USE_CXX11_ABI=0\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "csrc/flash_attn/src/fmha_bwd_launch_template.h(64): warning #177-D: variable \"M\" was declared but never referenced\n",
            "\n",
            "csrc/flash_attn/src/fmha_bwd_launch_template.h(64): warning #177-D: variable \"M\" was declared but never referenced\n",
            "\n",
            "ptxas info    : 17 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    24 bytes stack frame, 32 bytes spill stores, 36 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    64 bytes stack frame, 88 bytes spill stores, 84 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    40 bytes stack frame, 56 bytes spill stores, 48 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    24 bytes stack frame, 44 bytes spill stores, 56 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    120 bytes stack frame, 244 bytes spill stores, 232 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 26 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    64 bytes stack frame, 96 bytes spill stores, 112 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    96 bytes stack frame, 156 bytes spill stores, 148 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 16 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 16 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    80 bytes stack frame, 180 bytes spill stores, 228 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    144 bytes stack frame, 352 bytes spill stores, 348 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 250 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 240 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 254 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 249 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 251 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfEEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfEEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 26 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 245 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 242 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 244 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 249 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 243 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 245 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    40 bytes stack frame, 64 bytes spill stores, 72 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 208 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 208 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 200 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 204 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 210 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 218 registers\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 26 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 212 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 167 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 230 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 232 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 220 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 224 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 230 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 240 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    24 bytes stack frame, 32 bytes spill stores, 36 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    64 bytes stack frame, 88 bytes spill stores, 84 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    40 bytes stack frame, 56 bytes spill stores, 48 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    24 bytes stack frame, 44 bytes spill stores, 56 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    120 bytes stack frame, 244 bytes spill stores, 232 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 25 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    64 bytes stack frame, 96 bytes spill stores, 112 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    96 bytes stack frame, 156 bytes spill stores, 148 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 16 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 16 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    80 bytes stack frame, 180 bytes spill stores, 228 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    144 bytes stack frame, 352 bytes spill stores, 348 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 250 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 240 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 254 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 249 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 251 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 25 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 245 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 242 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 244 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 251 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 243 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 245 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    40 bytes stack frame, 64 bytes spill stores, 72 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 208 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 208 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 200 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 204 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 210 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 218 registers\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 25 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 212 registers\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 167 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 230 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 232 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 220 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 224 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 230 registers\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 240 registers\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "csrc/flash_attn/src/fmha_bwd_launch_template.h(64): warning #177-D: variable \"M\" was declared but never referenced\n",
            "\n",
            "csrc/flash_attn/src/fmha_bwd_launch_template.h(64): warning #177-D: variable \"M\" was declared but never referenced\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "csrc/flash_attn/src/fmha_bwd_launch_template.h(64): warning #177-D: variable \"M\" was declared but never referenced\n",
            "\n",
            "csrc/flash_attn/src/fmha_bwd_launch_template.h(64): warning #177-D: variable \"M\" was declared but never referenced\n",
            "\n",
            "ptxas info    : 17 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 20 bytes spill stores, 36 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    40 bytes stack frame, 64 bytes spill stores, 64 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 16 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    32 bytes stack frame, 48 bytes spill stores, 96 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    128 bytes stack frame, 276 bytes spill stores, 280 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 25 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    64 bytes stack frame, 100 bytes spill stores, 116 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    104 bytes stack frame, 168 bytes spill stores, 160 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 20 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    32 bytes stack frame, 40 bytes spill stores, 32 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    80 bytes stack frame, 192 bytes spill stores, 240 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    168 bytes stack frame, 396 bytes spill stores, 384 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 249 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 247 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 4 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfEEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfEEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 25 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 245 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 4 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 20 bytes spill stores, 28 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    56 bytes stack frame, 80 bytes spill stores, 92 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 165 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 190 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 200 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 206 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 196 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 198 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 206 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 224 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 25 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 194 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 196 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 206 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 212 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 202 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 204 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 214 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 222 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 20 bytes spill stores, 36 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    40 bytes stack frame, 64 bytes spill stores, 64 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 16 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    32 bytes stack frame, 48 bytes spill stores, 96 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    128 bytes stack frame, 276 bytes spill stores, 280 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 23 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    64 bytes stack frame, 100 bytes spill stores, 116 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    104 bytes stack frame, 168 bytes spill stores, 160 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 20 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    32 bytes stack frame, 40 bytes spill stores, 32 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    80 bytes stack frame, 192 bytes spill stores, 240 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    168 bytes stack frame, 396 bytes spill stores, 384 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 249 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 247 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 4 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 23 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 252 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 245 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 4 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 20 bytes spill stores, 28 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    56 bytes stack frame, 80 bytes spill stores, 92 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 165 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 190 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 200 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 206 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 196 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 198 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 206 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 224 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 23 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 194 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 196 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 206 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 212 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 202 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 204 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 214 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 222 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : 21457 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params\n",
            "    24 bytes stack frame, 32 bytes spill stores, 24 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    72 bytes stack frame, 120 bytes spill stores, 136 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    56 bytes stack frame, 100 bytes spill stores, 92 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    48 bytes stack frame, 52 bytes spill stores, 56 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    32 bytes stack frame, 48 bytes spill stores, 40 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    80 bytes stack frame, 236 bytes spill stores, 284 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    136 bytes stack frame, 296 bytes spill stores, 280 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 33 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 20 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params\n",
            "    24 bytes stack frame, 28 bytes spill stores, 24 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    80 bytes stack frame, 152 bytes spill stores, 160 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    128 bytes stack frame, 220 bytes spill stores, 204 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    56 bytes stack frame, 64 bytes spill stores, 64 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    32 bytes stack frame, 40 bytes spill stores, 32 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    96 bytes stack frame, 280 bytes spill stores, 328 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    184 bytes stack frame, 492 bytes spill stores, 476 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 12 bytes spill stores, 20 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    24 bytes stack frame, 20 bytes spill stores, 24 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfEEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfEEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 33 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 251 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    24 bytes stack frame, 28 bytes spill stores, 44 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    40 bytes stack frame, 48 bytes spill stores, 48 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    32 bytes stack frame, 56 bytes spill stores, 96 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    64 bytes stack frame, 96 bytes spill stores, 124 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 175 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 196 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 212 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 220 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 200 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 202 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 212 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 234 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfEEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 33 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 200 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 200 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 214 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 222 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 206 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 208 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 220 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E6__halfELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 228 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params\n",
            "    40 bytes stack frame, 56 bytes spill stores, 52 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    80 bytes stack frame, 156 bytes spill stores, 180 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    72 bytes stack frame, 120 bytes spill stores, 112 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    64 bytes stack frame, 80 bytes spill stores, 88 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    40 bytes stack frame, 52 bytes spill stores, 48 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    112 bytes stack frame, 444 bytes spill stores, 512 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    160 bytes stack frame, 508 bytes spill stores, 492 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 38 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 20 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params\n",
            "    40 bytes stack frame, 44 bytes spill stores, 36 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    88 bytes stack frame, 172 bytes spill stores, 196 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    80 bytes stack frame, 132 bytes spill stores, 124 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    72 bytes stack frame, 84 bytes spill stores, 96 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    48 bytes stack frame, 60 bytes spill stores, 56 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    120 bytes stack frame, 432 bytes spill stores, 488 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    216 bytes stack frame, 592 bytes spill stores, 572 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 12 bytes spill stores, 20 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 4 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 20 bytes spill stores, 36 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    48 bytes stack frame, 48 bytes spill stores, 52 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 38 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    24 bytes stack frame, 20 bytes spill stores, 36 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    16 bytes stack frame, 16 bytes spill stores, 20 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 16 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    24 bytes stack frame, 32 bytes spill stores, 68 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi8ELj256E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    96 bytes stack frame, 124 bytes spill stores, 148 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 175 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 196 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 212 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 220 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 202 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 204 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 212 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb0ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 236 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z24fmha_bwd_dot_do_o_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 38 registers, 640 bytes cmem[0]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 200 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z40fmha_bwd_q_dk_dv_loop_seqparallel_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 200 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 214 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi2EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 222 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 208 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELi1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 210 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb0ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 220 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z29fmha_bwd_dq_dk_dv_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi8ELj8E13__nv_bfloat16ELb1ELb1ELin1EEv17FMHA_dgrad_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 240 registers, 640 bytes cmem[0], 16 bytes cmem[2]\n",
            "/usr/local/cuda/bin/nvcc -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/src -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/cutlass/include -I/usr/local/lib/python3.10/dist-packages/torch/include -I/usr/local/lib/python3.10/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.10/dist-packages/torch/include/TH -I/usr/local/lib/python3.10/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.10 -c csrc/flash_attn/src/fmha_fwd_hdim128.cu -o build/temp.linux-x86_64-cpython-310/csrc/flash_attn/src/fmha_fwd_hdim128.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -O3 -std=c++17 -U__CUDA_NO_HALF_OPERATORS__ -U__CUDA_NO_HALF_CONVERSIONS__ -U__CUDA_NO_HALF2_OPERATORS__ -U__CUDA_NO_BFLOAT16_CONVERSIONS__ --expt-relaxed-constexpr --expt-extended-lambda --use_fast_math --ptxas-options=-v -lineinfo -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_90,code=sm_90 --threads 4 -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\\\"_gcc\\\" -DPYBIND11_STDLIB=\\\"_libstdcpp\\\" -DPYBIND11_BUILD_ABI=\\\"_cxxabi1011\\\" -DTORCH_EXTENSION_NAME=flash_attn_cuda -D_GLIBCXX_USE_CXX11_ABI=0\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "ptxas info    : 9 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 12 bytes spill stores, 20 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 254 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 12 bytes spill stores, 20 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 254 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "ptxas info    : 9 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    16 bytes stack frame, 16 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 254 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    16 bytes stack frame, 16 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 254 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : 11186 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 24 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi128ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "/usr/local/cuda/bin/nvcc -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/src -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/cutlass/include -I/usr/local/lib/python3.10/dist-packages/torch/include -I/usr/local/lib/python3.10/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.10/dist-packages/torch/include/TH -I/usr/local/lib/python3.10/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.10 -c csrc/flash_attn/src/fmha_fwd_hdim32.cu -o build/temp.linux-x86_64-cpython-310/csrc/flash_attn/src/fmha_fwd_hdim32.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -O3 -std=c++17 -U__CUDA_NO_HALF_OPERATORS__ -U__CUDA_NO_HALF_CONVERSIONS__ -U__CUDA_NO_HALF2_OPERATORS__ -U__CUDA_NO_BFLOAT16_CONVERSIONS__ --expt-relaxed-constexpr --expt-extended-lambda --use_fast_math --ptxas-options=-v -lineinfo -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_90,code=sm_90 --threads 4 -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\\\"_gcc\\\" -DPYBIND11_STDLIB=\\\"_libstdcpp\\\" -DPYBIND11_BUILD_ABI=\\\"_cxxabi1011\\\" -DTORCH_EXTENSION_NAME=flash_attn_cuda -D_GLIBCXX_USE_CXX11_ABI=0\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "ptxas info    : 17 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 180 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 194 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 202 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 191 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 188 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 218 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 234 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 127 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 132 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 143 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 150 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 133 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 142 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 153 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 158 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 180 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 194 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 202 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 191 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 188 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 218 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 234 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 127 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 132 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 143 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 150 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 133 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 142 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 153 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 158 registers\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "ptxas info    : 17 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 178 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 193 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 202 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 194 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 232 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 220 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 126 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 128 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 136 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 140 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 125 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 127 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 145 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 145 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 178 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 193 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 202 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 194 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 232 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 220 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 126 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 128 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 136 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 140 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 125 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 127 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 145 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 145 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : 22250 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 191 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 203 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 198 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 174 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 219 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 219 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 114 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 128 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 128 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 128 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 119 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 128 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 127 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 130 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 178 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 175 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 192 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 203 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 188 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 206 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 219 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 221 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 124 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 129 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 131 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 134 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 126 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 128 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 130 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi32ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 132 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "/usr/local/cuda/bin/nvcc -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/src -I/content/flash-attention/flash-attention/flash-attention/flash-attention/csrc/flash_attn/cutlass/include -I/usr/local/lib/python3.10/dist-packages/torch/include -I/usr/local/lib/python3.10/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.10/dist-packages/torch/include/TH -I/usr/local/lib/python3.10/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.10 -c csrc/flash_attn/src/fmha_fwd_hdim64.cu -o build/temp.linux-x86_64-cpython-310/csrc/flash_attn/src/fmha_fwd_hdim64.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr --compiler-options '-fPIC' -O3 -std=c++17 -U__CUDA_NO_HALF_OPERATORS__ -U__CUDA_NO_HALF_CONVERSIONS__ -U__CUDA_NO_HALF2_OPERATORS__ -U__CUDA_NO_BFLOAT16_CONVERSIONS__ --expt-relaxed-constexpr --expt-extended-lambda --use_fast_math --ptxas-options=-v -lineinfo -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_90,code=sm_90 --threads 4 -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\\\"_gcc\\\" -DPYBIND11_STDLIB=\\\"_libstdcpp\\\" -DPYBIND11_BUILD_ABI=\\\"_cxxabi1011\\\" -DTORCH_EXTENSION_NAME=flash_attn_cuda -D_GLIBCXX_USE_CXX11_ABI=0\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "ptxas info    : 17 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    16 bytes stack frame, 24 bytes spill stores, 32 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    24 bytes stack frame, 28 bytes spill stores, 24 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 20 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 167 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 170 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 164 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 164 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 177 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    16 bytes stack frame, 24 bytes spill stores, 32 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    24 bytes stack frame, 28 bytes spill stores, 24 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 20 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 167 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 170 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 164 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 164 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_90'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 177 registers\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=false, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(77): here\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/util/irange.h(54): warning #186-D: pointless comparison of unsigned integer with zero\n",
            "          detected during:\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator==(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "(61): here\n",
            "            instantiation of \"__nv_bool c10::detail::integer_iterator<I, one_sided, <unnamed>>::operator!=(const c10::detail::integer_iterator<I, one_sided, <unnamed>> &) const [with I=size_t, one_sided=true, <unnamed>=0]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2327): here\n",
            "            instantiation of \"__nv_bool c10::TensorImpl::SetDimsTemplate(c10::ArrayRef<T>) [with T=int64_t, <unnamed>=void]\" \n",
            "/usr/local/lib/python3.10/dist-packages/torch/include/c10/core/TensorImpl.h(2337): here\n",
            "\n",
            "ptxas info    : 17 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 12 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    24 bytes stack frame, 28 bytes spill stores, 32 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 254 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    24 bytes stack frame, 24 bytes spill stores, 32 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 167 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 172 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 165 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 174 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 167 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 175 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 12 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    24 bytes stack frame, 28 bytes spill stores, 32 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 254 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    24 bytes stack frame, 24 bytes spill stores, 32 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 167 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 172 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 165 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 174 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 167 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_80'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 175 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : 22250 bytes gmem\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 254 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 8 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 20 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    24 bytes stack frame, 20 bytes spill stores, 20 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    16 bytes stack frame, 12 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 167 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 166 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E6__halfELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 178 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 8 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    32 bytes stack frame, 28 bytes spill stores, 36 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi256ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    8 bytes stack frame, 4 bytes spill stores, 12 bytes spill loads\n",
            "ptxas info    : Used 255 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 184 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 176 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb0ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 171 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb0ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb0EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 176 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "ptxas info    : Compiling entry function '_Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params' for 'sm_75'\n",
            "ptxas info    : Function properties for _Z20fmha_fwd_loop_kernelI18FMHA_kernel_traitsILi128ELi64ELi16ELi1ELi4ELj8E13__nv_bfloat16ELb1ELb1ELb1EEv17FMHA_fprop_params\n",
            "    0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads\n",
            "ptxas info    : Used 168 registers, 576 bytes cmem[0], 16 bytes cmem[2]\n",
            "x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -g -fwrapv -O2 build/temp.linux-x86_64-cpython-310/csrc/flash_attn/fmha_api.o build/temp.linux-x86_64-cpython-310/csrc/flash_attn/src/fmha_block_dgrad_fp16_kernel_loop.sm80.o build/temp.linux-x86_64-cpython-310/csrc/flash_attn/src/fmha_block_fprop_fp16_kernel.sm80.o build/temp.linux-x86_64-cpython-310/csrc/flash_attn/src/fmha_bwd_hdim128.o build/temp.linux-x86_64-cpython-310/csrc/flash_attn/src/fmha_bwd_hdim32.o build/temp.linux-x86_64-cpython-310/csrc/flash_attn/src/fmha_bwd_hdim64.o build/temp.linux-x86_64-cpython-310/csrc/flash_attn/src/fmha_fwd_hdim128.o build/temp.linux-x86_64-cpython-310/csrc/flash_attn/src/fmha_fwd_hdim32.o build/temp.linux-x86_64-cpython-310/csrc/flash_attn/src/fmha_fwd_hdim64.o -L/usr/local/lib/python3.10/dist-packages/torch/lib -L/usr/local/cuda/lib64 -L/usr/lib/x86_64-linux-gnu -lc10 -ltorch -ltorch_cpu -ltorch_python -lcudart -lc10_cuda -ltorch_cuda -o build/lib.linux-x86_64-cpython-310/flash_attn_cuda.cpython-310-x86_64-linux-gnu.so\n",
            "creating build/bdist.linux-x86_64\n",
            "creating build/bdist.linux-x86_64/egg\n",
            "creating build/bdist.linux-x86_64/egg/flash_attn\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/bert_padding.py -> build/bdist.linux-x86_64/egg/flash_attn\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/flash_blocksparse_attention.py -> build/bdist.linux-x86_64/egg/flash_attn\n",
            "creating build/bdist.linux-x86_64/egg/flash_attn/losses\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/losses/__init__.py -> build/bdist.linux-x86_64/egg/flash_attn/losses\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/losses/cross_entropy.py -> build/bdist.linux-x86_64/egg/flash_attn/losses\n",
            "creating build/bdist.linux-x86_64/egg/flash_attn/layers\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/layers/__init__.py -> build/bdist.linux-x86_64/egg/flash_attn/layers\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/layers/rotary.py -> build/bdist.linux-x86_64/egg/flash_attn/layers\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/layers/patch_embed.py -> build/bdist.linux-x86_64/egg/flash_attn/layers\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/flash_attn_interface.py -> build/bdist.linux-x86_64/egg/flash_attn\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/flash_blocksparse_attn_interface.py -> build/bdist.linux-x86_64/egg/flash_attn\n",
            "creating build/bdist.linux-x86_64/egg/flash_attn/utils\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/utils/distributed.py -> build/bdist.linux-x86_64/egg/flash_attn/utils\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/utils/__init__.py -> build/bdist.linux-x86_64/egg/flash_attn/utils\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/utils/generation.py -> build/bdist.linux-x86_64/egg/flash_attn/utils\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/utils/benchmark.py -> build/bdist.linux-x86_64/egg/flash_attn/utils\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/utils/pretrained.py -> build/bdist.linux-x86_64/egg/flash_attn/utils\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/flash_attn_triton_og.py -> build/bdist.linux-x86_64/egg/flash_attn\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/flash_attention.py -> build/bdist.linux-x86_64/egg/flash_attn\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/__init__.py -> build/bdist.linux-x86_64/egg/flash_attn\n",
            "creating build/bdist.linux-x86_64/egg/flash_attn/ops\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/ops/activations.py -> build/bdist.linux-x86_64/egg/flash_attn/ops\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/ops/fused_dense.py -> build/bdist.linux-x86_64/egg/flash_attn/ops\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/ops/__init__.py -> build/bdist.linux-x86_64/egg/flash_attn/ops\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/ops/rms_norm.py -> build/bdist.linux-x86_64/egg/flash_attn/ops\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/ops/layer_norm.py -> build/bdist.linux-x86_64/egg/flash_attn/ops\n",
            "creating build/bdist.linux-x86_64/egg/flash_attn/models\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/models/gptj.py -> build/bdist.linux-x86_64/egg/flash_attn/models\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/models/vit.py -> build/bdist.linux-x86_64/egg/flash_attn/models\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/models/bert.py -> build/bdist.linux-x86_64/egg/flash_attn/models\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/models/gpt.py -> build/bdist.linux-x86_64/egg/flash_attn/models\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/models/llama.py -> build/bdist.linux-x86_64/egg/flash_attn/models\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/models/__init__.py -> build/bdist.linux-x86_64/egg/flash_attn/models\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/models/opt.py -> build/bdist.linux-x86_64/egg/flash_attn/models\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/models/gpt_neox.py -> build/bdist.linux-x86_64/egg/flash_attn/models\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/flash_attn_triton.py -> build/bdist.linux-x86_64/egg/flash_attn\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/fused_softmax.py -> build/bdist.linux-x86_64/egg/flash_attn\n",
            "creating build/bdist.linux-x86_64/egg/flash_attn/modules\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/modules/mha.py -> build/bdist.linux-x86_64/egg/flash_attn/modules\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/modules/block.py -> build/bdist.linux-x86_64/egg/flash_attn/modules\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/modules/mlp.py -> build/bdist.linux-x86_64/egg/flash_attn/modules\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/modules/__init__.py -> build/bdist.linux-x86_64/egg/flash_attn/modules\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn/modules/embedding.py -> build/bdist.linux-x86_64/egg/flash_attn/modules\n",
            "copying build/lib.linux-x86_64-cpython-310/flash_attn_cuda.cpython-310-x86_64-linux-gnu.so -> build/bdist.linux-x86_64/egg\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/bert_padding.py to bert_padding.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/flash_blocksparse_attention.py to flash_blocksparse_attention.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/losses/__init__.py to __init__.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/losses/cross_entropy.py to cross_entropy.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/layers/__init__.py to __init__.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/layers/rotary.py to rotary.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/layers/patch_embed.py to patch_embed.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/flash_attn_interface.py to flash_attn_interface.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/flash_blocksparse_attn_interface.py to flash_blocksparse_attn_interface.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/utils/distributed.py to distributed.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/utils/__init__.py to __init__.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/utils/generation.py to generation.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/utils/benchmark.py to benchmark.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/utils/pretrained.py to pretrained.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/flash_attn_triton_og.py to flash_attn_triton_og.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/flash_attention.py to flash_attention.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/__init__.py to __init__.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/ops/activations.py to activations.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/ops/fused_dense.py to fused_dense.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/ops/__init__.py to __init__.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/ops/rms_norm.py to rms_norm.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/ops/layer_norm.py to layer_norm.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/models/gptj.py to gptj.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/models/vit.py to vit.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/models/bert.py to bert.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/models/gpt.py to gpt.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/models/llama.py to llama.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/models/__init__.py to __init__.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/models/opt.py to opt.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/models/gpt_neox.py to gpt_neox.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/flash_attn_triton.py to flash_attn_triton.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/fused_softmax.py to fused_softmax.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/modules/mha.py to mha.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/modules/block.py to block.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/modules/mlp.py to mlp.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/modules/__init__.py to __init__.cpython-310.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn/modules/embedding.py to embedding.cpython-310.pyc\n",
            "creating stub loader for flash_attn_cuda.cpython-310-x86_64-linux-gnu.so\n",
            "byte-compiling build/bdist.linux-x86_64/egg/flash_attn_cuda.py to flash_attn_cuda.cpython-310.pyc\n",
            "creating build/bdist.linux-x86_64/egg/EGG-INFO\n",
            "copying flash_attn.egg-info/PKG-INFO -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
            "copying flash_attn.egg-info/SOURCES.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
            "copying flash_attn.egg-info/dependency_links.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
            "copying flash_attn.egg-info/requires.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
            "copying flash_attn.egg-info/top_level.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
            "writing build/bdist.linux-x86_64/egg/EGG-INFO/native_libs.txt\n",
            "zip_safe flag not set; analyzing archive contents...\n",
            "__pycache__.flash_attn_cuda.cpython-310: module references __file__\n",
            "creating dist\n",
            "creating 'dist/flash_attn-1.0.8-py3.10-linux-x86_64.egg' and adding 'build/bdist.linux-x86_64/egg' to it\n",
            "removing 'build/bdist.linux-x86_64/egg' (and everything under it)\n",
            "Processing flash_attn-1.0.8-py3.10-linux-x86_64.egg\n",
            "removing '/usr/local/lib/python3.10/dist-packages/flash_attn-1.0.8-py3.10-linux-x86_64.egg' (and everything under it)\n",
            "creating /usr/local/lib/python3.10/dist-packages/flash_attn-1.0.8-py3.10-linux-x86_64.egg\n",
            "Extracting flash_attn-1.0.8-py3.10-linux-x86_64.egg to /usr/local/lib/python3.10/dist-packages\n",
            "flash-attn 1.0.8 is already the active version in easy-install.pth\n",
            "\n",
            "Installed /usr/local/lib/python3.10/dist-packages/flash_attn-1.0.8-py3.10-linux-x86_64.egg\n",
            "Processing dependencies for flash-attn==1.0.8\n",
            "Searching for ninja==1.11.1\n",
            "Best match: ninja 1.11.1\n",
            "Processing ninja-1.11.1-py3.10-linux-x86_64.egg\n",
            "ninja 1.11.1 is already the active version in easy-install.pth\n",
            "Installing ninja script to /usr/local/bin\n",
            "\n",
            "Using /usr/local/lib/python3.10/dist-packages/ninja-1.11.1-py3.10-linux-x86_64.egg\n",
            "Searching for packaging==23.1\n",
            "Best match: packaging 23.1\n",
            "Adding packaging 23.1 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.10/dist-packages\n",
            "Searching for einops==0.7.0rc1\n",
            "Best match: einops 0.7.0rc1\n",
            "Processing einops-0.7.0rc1-py3.10.egg\n",
            "einops 0.7.0rc1 is already the active version in easy-install.pth\n",
            "\n",
            "Using /usr/local/lib/python3.10/dist-packages/einops-0.7.0rc1-py3.10.egg\n",
            "Searching for torch==2.0.1+cu118\n",
            "Best match: torch 2.0.1+cu118\n",
            "Adding torch 2.0.1+cu118 to easy-install.pth file\n",
            "Installing convert-caffe2-to-onnx script to /usr/local/bin\n",
            "Installing convert-onnx-to-caffe2 script to /usr/local/bin\n",
            "Installing torchrun script to /usr/local/bin\n",
            "\n",
            "Using /usr/local/lib/python3.10/dist-packages\n",
            "Searching for triton==2.0.0\n",
            "Best match: triton 2.0.0\n",
            "Adding triton 2.0.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.10/dist-packages\n",
            "Searching for Jinja2==3.1.2\n",
            "Best match: Jinja2 3.1.2\n",
            "Adding Jinja2 3.1.2 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.10/dist-packages\n",
            "Searching for networkx==3.1\n",
            "Best match: networkx 3.1\n",
            "Adding networkx 3.1 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.10/dist-packages\n",
            "Searching for sympy==1.11.1\n",
            "Best match: sympy 1.11.1\n",
            "Adding sympy 1.11.1 to easy-install.pth file\n",
            "Installing isympy script to /usr/local/bin\n",
            "\n",
            "Using /usr/local/lib/python3.10/dist-packages\n",
            "Searching for typing-extensions==4.6.3\n",
            "Best match: typing-extensions 4.6.3\n",
            "Adding typing-extensions 4.6.3 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.10/dist-packages\n",
            "Searching for filelock==3.12.2\n",
            "Best match: filelock 3.12.2\n",
            "Adding filelock 3.12.2 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.10/dist-packages\n",
            "Searching for lit==16.0.6\n",
            "Best match: lit 16.0.6\n",
            "Adding lit 16.0.6 to easy-install.pth file\n",
            "Installing lit script to /usr/local/bin\n",
            "\n",
            "Using /usr/local/lib/python3.10/dist-packages\n",
            "Searching for cmake==3.25.2\n",
            "Best match: cmake 3.25.2\n",
            "Adding cmake 3.25.2 to easy-install.pth file\n",
            "Installing cmake script to /usr/local/bin\n",
            "Installing cpack script to /usr/local/bin\n",
            "Installing ctest script to /usr/local/bin\n",
            "\n",
            "Using /usr/local/lib/python3.10/dist-packages\n",
            "Searching for MarkupSafe==2.1.3\n",
            "Best match: MarkupSafe 2.1.3\n",
            "Adding MarkupSafe 2.1.3 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.10/dist-packages\n",
            "Searching for mpmath==1.3.0\n",
            "Best match: mpmath 1.3.0\n",
            "Adding mpmath 1.3.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.10/dist-packages\n",
            "Finished processing dependencies for flash-attn==1.0.8\n",
            "[Errno 2] No such file or directory: '.. # Go back to the parent directory after the installation'\n",
            "/content/flash-attention/flash-attention/flash-attention/flash-attention\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "ModuleNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-11-51a259bcb0a9>\u001b[0m in \u001b[0;36m<cell line: 15>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunctional\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mflash_attn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflash_attention\u001b[0m \u001b[0;32mimport\u001b[0m  \u001b[0mFlashMHA\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtorchscale\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mXPOS\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mRelativePositionBias\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/flash-attention/flash_attn/flash_attention.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0meinops\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mrearrange\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mflash_attn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflash_attn_interface\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mflash_attn_unpadded_qkvpacked_func\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'einops'",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ],
          "errorDetails": {
            "actions": [
              {
                "action": "open_url",
                "actionText": "Open Examples",
                "url": "/notebooks/snippets/importing_libraries.ipynb"
              }
            ]
          }
        }
      ],
      "source": [
        "\n",
        "!pip install torch\n",
        "!pip install einops\n",
        "!pip install torchscale\n",
        "\n",
        "!git clone https://github.com/HazyResearch/flash-attention.git\n",
        "%cd flash-attention\n",
        "!python setup.py install\n",
        "%cd ..  # Go back to the parent directory after the installation\n",
        "\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "from flash_attn.flash_attention import  FlashMHA\n",
        "\n",
        "from torchscale import XPOS, RelativePositionBias\n",
        "\n",
        "# Replace this with your correct GPU device\n",
        "device = \"cuda:0\"\n",
        "dtype=torch.float16\n",
        "\n",
        "#add alibi, qk layer norm, one write head, multihway,\n",
        "class DilatedAttention(nn.Module):\n",
        "    \"\"\"\n",
        "    Dilated Attention Module.\n",
        "\n",
        "    Arguments:\n",
        "        d_model: The dimension of the attention layers.\n",
        "        num_heads: The number of attention heads.\n",
        "        dilation_rate: The dilation rate for dilated attention.\n",
        "        segment_size: The segment size for dilated attention.\n",
        "        dropout (optional): The dropout probability. Default: 0.0\n",
        "        casual (optional): If set to True, the attention mechanism is casual. Default: False\n",
        "        use_xpos (optional): If set to True, xpos is used for positional encoding. Default: False\n",
        "        use_rel_pos_bias (optional): If set to True, relative position bias is used in the attention mechanism. Default: False\n",
        "\n",
        "    Usage:\n",
        "        The `DilatedAttention` class can be used as a module for neural networks and is especially suited for transformer architectures.\n",
        "\n",
        "        Example:\n",
        "            attention = DilatedAttention(d_model=512, num_heads=8, dilation_rate=2, segment_size=64, use_xpos=True, use_rel_pos_bias=True)\n",
        "            output = attention(input_tensor)\n",
        "\n",
        "        This will return the output tensor after applying dilated attention. The `use_xpos` and `use_rel_pos_bias` parameters allow for switching on positional encoding and relative positional bias respectively.\n",
        "    \"\"\"\n",
        "    def __init__(self, d_model, num_heads, dilation_rate, segment_size, dropout=0.0, casual=False, use_xpos=False, use_rel_pos_bias=False):\n",
        "        super(DilatedAttention, self).__init__()\n",
        "        self.d_model = d_model\n",
        "        self.num_heads = num_heads\n",
        "\n",
        "        self.dilation_rate = dilation_rate\n",
        "        self.segment_size = segment_size\n",
        "\n",
        "        self.attention = FlashMHA(embed_dim=d_model, num_heads=num_heads, device=device, dtype=dtype)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        self.casual = casual\n",
        "\n",
        "        self.use_xpos = use_xpos\n",
        "        self.use_rel_pos_bias = use_rel_pos_bias\n",
        "\n",
        "        if use_xpos:\n",
        "            self.xpos = XPOS(head_dim=d_model//num_heads)\n",
        "        if use_rel_pos_bias:\n",
        "            self.relative_bias = RelativePositionBias(num_buckets=32, max_distance=128, n_heads=num_heads)\n",
        "\n",
        "    def get_mask(self, i, j):\n",
        "        return torch.ones((i, j), device=device, dtype=torch.bool).triu(j - i + 2)\n",
        "\n",
        "    def forward(self, x):\n",
        "        batch_size, seq_len, _ = x.shape\n",
        "\n",
        "        if self.use_xpos:\n",
        "            x = self.xpos(x)\n",
        "\n",
        "        # Split and sparsify\n",
        "        x = x.view(batch_size, -1, self.segment_size, self.d_model)\n",
        "        x = x[:, :, :: self.dilation_rate, :]\n",
        "\n",
        "        # Perform attention\n",
        "        attn_output, _ = self.attention(x, x, x)\n",
        "\n",
        "        if self.use_rel_pos_bias:\n",
        "            attn_output += self.relative_bias(batch_size, attn_output.size(1), attn_output.size(1))\n",
        "\n",
        "        # if casual create a mask and apply to the output\n",
        "        if self.casual:\n",
        "            mask = self.get_mask(attn_output.size(1), attn_output.size(1))\n",
        "            attn_output = attn_output.masked_fill(mask, float('-inf'))\n",
        "\n",
        "        # apply dropout\n",
        "        attn_output = self.dropout(attn_output)\n",
        "\n",
        "        # Scatter and concatenate\n",
        "        attn_output = attn_output.view(batch_size, -1, self.d_model)\n",
        "        return attn_output\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6QJNBSLSx-aX"
      },
      "outputs": [],
      "source": [
        "import time\n",
        "import unittest\n",
        "import torch\n",
        "\n",
        "# from LongNet import DilatedAttention, MultiModalDilationAttention\n",
        "\n",
        "class TestDilatedAttention(unittest.TestCase):\n",
        "\n",
        "    def test_output_shape(self):\n",
        "        # Setup\n",
        "        input_tensor = torch.randn(2, 128, 512)\n",
        "        dilated_attention = DilatedAttention(512, 8, 2, 64)\n",
        "\n",
        "        # Action\n",
        "        output = dilated_attention(input_tensor)\n",
        "\n",
        "        # Assert\n",
        "        self.assertEqual(output.shape, (2, 128, 512))\n",
        "\n",
        "    def test_xpos(self):\n",
        "        # Setup\n",
        "        input_tensor = torch.randn(2, 128, 512)\n",
        "        dilated_attention = DilatedAttention(512, 8, 2, 64, use_xpos=True)\n",
        "\n",
        "        # Action\n",
        "        output = dilated_attention(input_tensor)\n",
        "\n",
        "        # Assert\n",
        "        self.assertEqual(output.shape, (2, 128, 512))\n",
        "\n",
        "    def test_relative_position_bias(self):\n",
        "        # Setup\n",
        "        input_tensor = torch.randn(2, 128, 512)\n",
        "        dilated_attention = DilatedAttention(512, 8, 2, 64, use_rel_pos_bias=True)\n",
        "\n",
        "        # Action\n",
        "        output = dilated_attention(input_tensor)\n",
        "\n",
        "        # Assert\n",
        "        self.assertEqual(output.shape, (2, 128, 512))\n",
        "\n",
        "\n",
        "    def test_attention_consistency(self):\n",
        "        # Setup\n",
        "        input_tensor = torch.randn(2, 128, 512)\n",
        "        dilated_attention = DilatedAttention(512, 8, 2, 64)\n",
        "\n",
        "        # Action\n",
        "        output = dilated_attention(input_tensor)\n",
        "\n",
        "        # Assert\n",
        "        self.assertTrue((output.std(dim=-1) > 0).all())\n",
        "\n",
        "    def test_speed(self):\n",
        "        # Setup\n",
        "        input_tensor = torch.randn(2, 1024, 512)\n",
        "        dilated_attention = DilatedAttention(512, 8, 2, 64)\n",
        "\n",
        "        # Action\n",
        "        start_time = time.time()\n",
        "        output = dilated_attention(input_tensor)\n",
        "        end_time = time.time()\n",
        "\n",
        "        # Assert\n",
        "        self.assertLess(end_time - start_time, 1)\n",
        "\n",
        "    def test_gradient_flow(self):\n",
        "        # Setup\n",
        "        input_tensor = torch.randn(2, 128, 512, requires_grad=True)\n",
        "        dilated_attention = DilatedAttention(512, 8, 2, 64)\n",
        "\n",
        "        # Action\n",
        "        output = dilated_attention(input_tensor)\n",
        "        output.sum().backward()\n",
        "        grad_norm = input_tensor.grad.norm().item()\n",
        "\n",
        "        # Assert\n",
        "        self.assertLess(grad_norm, 1e6)\n",
        "        self.assertGreater(grad_norm, 1e-6)\n",
        "\n",
        "    def test_scaling(self):\n",
        "        input_tensor = torch.randn(2, 1024, 512)\n",
        "        dilated_attention = DilatedAttention(512, 8, 2, 64)\n",
        "        start_time = time.time()\n",
        "        _ = dilated_attention(input_tensor)\n",
        "        time_for_1024 = time.time() - start_time\n",
        "\n",
        "        input_tensor = torch.randn(2, 2048, 512)\n",
        "        start_time = time.time()\n",
        "        _ = dilated_attention(input_tensor)\n",
        "        time_for_2048 = time.time() - start_time\n",
        "\n",
        "        self.assertLessEqual(time_for_2048/time_for_1024, 2)\n",
        "\n",
        "    def test_reproducibility(self):\n",
        "        torch.manual_seed(0)\n",
        "        input_tensor = torch.randn(2, 128, 512)\n",
        "        dilated_attention = DilatedAttention(512, 8, 2, 64)\n",
        "        output1 = dilated_attention(input_tensor)\n",
        "\n",
        "        torch.manual_seed(0)\n",
        "        input_tensor = torch.randn(2, 128, 512)\n",
        "        dilated_attention = DilatedAttention(512, 8, 2, 64)\n",
        "        output2 = dilated_attention(input_tensor)\n",
        "\n",
        "        self.assertTrue(torch.allclose(output1, output2))\n",
        "\n",
        "    def test_attention_distribution(self):\n",
        "        input_tensor = torch.randn(2, 128, 512)\n",
        "        dilated_attention = DilatedAttention(512, 8, 2, 64)\n",
        "        _, attn_weights = dilated_attention(input_tensor)\n",
        "\n",
        "        self.assertTrue(torch.allclose(attn_weights.sum(dim=-1), torch.tensor(1.)))\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wzPNAUpvzpml"
      },
      "outputs": [],
      "source": [
        "import timeit\n",
        "import torch\n",
        "\n",
        "#model config\n",
        "d_model = 512\n",
        "num_heads = 8\n",
        "dilation_rate = 2\n",
        "segment_size = 64\n",
        "\n",
        "device = \"cuda:0\"\n",
        "dtype=torch.float16\n",
        "\n",
        "#input data\n",
        "batch_size = 32\n",
        "seq_len = 1024\n",
        "\n",
        "\n",
        "#create model and data\n",
        "model = DilatedAttention(d_model, num_heads, dilation_rate, segment_size).to(device)\n",
        "x = torch.randn((batch_size, seq_len, d_model), device=device, dtype=dtype)\n",
        "\n",
        "\n",
        "#test forward pass\n",
        "with torch.no_grad():\n",
        "    output = model(x)\n",
        "    print(f\"Output shape: {output.shape}\") # expected (batch_size, seq_Len)\n",
        "\n",
        "\n",
        "#benchmark model\n",
        "num_runs = 1000\n",
        "start_time = timeit.default_timer()\n",
        "for _ in range(num_runs):\n",
        "    model(x)\n",
        "\n",
        "elapsed_time = timeit.default_timer() - start_time\n",
        "print(f\"Average forward pass time: {elapsed_time / num_runs:.6f} seconds\")"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}