feat(deepseek-v4): support W8A8 INT8 output projection

.github/workflows/ci.yml

@@ -3,20 +3,7 @@
 # It includes linting and platform-specific testing stages.
 name: CI
 
-on:
-  schedule:
-    - cron: '0 18 * * *'  # daily at 2:00 AM CST (UTC+8)
-  pull_request:
-    branches: [main, devops]
-    paths-ignore:
-      - "**.md"
-      - "docs/**"
-      - "examples/**"
-      - "docker/**"
-      - "LICENSE"
-      - ".github/ISSUE_TEMPLATE/**"
-      - ".github/PULL_REQUEST_TEMPLATE.md"
-  workflow_dispatch:
+on: []  # disabled
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

examples/run_dsv4.sh

@@ -0,0 +1,14 @@
+export USE_FLAGGEMS=false
+export VLLM_USE_BREAKABLE_CUDAGRAPH=0
+
+vllm serve /public-nvme/models/DeepSeek-V4-Flash-0731-INT \
+  --trust-remote-code \
+  --kv-cache-dtype fp8 \
+  --block-size 256 \
+  --enable-expert-parallel \
+  --tensor-parallel-size 8 \
+  --tokenizer-mode deepseek_v4 \
+  --tool-call-parser deepseek_v4 \
+  --enable-auto-tool-choice \
+  --reasoning-parser deepseek_v4 \
+  --reasoning-config '{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'

tests/unit_tests/compilation/test_deepseek_v4_splitting_ops.py

@@ -0,0 +1,22 @@
+# Copyright (c) 2025 BAAI. All rights reserved.
+
+from vllm.config.compilation import CompilationConfig, CompilationMode
+
+import vllm_fl
+
+
+def test_register_deepseek_v4_fl_attention_as_splitting_op(monkeypatch):
+    attention_ops = ["vllm::deepseek_v4_attention"]
+    monkeypatch.setattr(CompilationConfig, "_attention_ops", attention_ops)
+
+    vllm_fl._register_compilation_splitting_ops()
+    vllm_fl._register_compilation_splitting_ops()
+
+    assert attention_ops == [
+        "vllm::deepseek_v4_attention",
+        "vllm::deepseek_v4_fl_attention",
+    ]
+
+    config = CompilationConfig(mode=CompilationMode.VLLM_COMPILE)
+    config.set_splitting_ops_for_v1("")
+    assert config.splitting_ops.count("vllm::deepseek_v4_fl_attention") == 1

tests/unit_tests/dispatch/test_cuda_backend_detection.py

@@ -0,0 +1,47 @@
+# Copyright (c) 2026 BAAI. All rights reserved.
+
+"""Tests for NVIDIA vendor backend detection on vLLM CUDA platforms."""
+
+from unittest.mock import Mock
+
+import torch
+from vllm import platforms
+
+from vllm_fl.dispatch.backends.vendor.cuda.cuda import CudaBackend
+from vllm_fl.dispatch.builtin_ops import _get_current_vendor_backend_dirs
+
+
+def test_in_tree_cuda_platform_selects_cuda_vendor_backend(monkeypatch):
+    platform = Mock()
+    platform.vendor_name = None
+    platform.device_name = "cuda"
+    platform.is_cuda.return_value = True
+    monkeypatch.setattr(platforms, "current_platform", platform)
+
+    assert _get_current_vendor_backend_dirs({"cuda", "ascend"}) == "cuda"
+
+
+def test_cuda_backend_available_for_in_tree_cuda_platform(monkeypatch):
+    platform = Mock()
+    platform.device_name = "cuda"
+    platform.is_cuda.return_value = True
+    monkeypatch.setattr(platforms, "current_platform", platform)
+    monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+    monkeypatch.setattr(torch.cuda, "device_count", lambda: 8)
+    monkeypatch.setattr(CudaBackend, "_available", None)
+
+    assert CudaBackend().is_available()
+
+
+def test_cuda_alike_platform_does_not_select_nvidia_backend(monkeypatch):
+    platform = Mock()
+    platform.vendor_name = None
+    platform.device_name = "cuda"
+    platform.is_cuda.return_value = False
+    monkeypatch.setattr(platforms, "current_platform", platform)
+    monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+    monkeypatch.setattr(torch.cuda, "device_count", lambda: 8)
+    monkeypatch.setattr(CudaBackend, "_available", None)
+
+    assert _get_current_vendor_backend_dirs({"cuda", "ascend"}) is None
+    assert not CudaBackend().is_available()

tests/unit_tests/dispatch/test_deepseek_v4_attention.py

@@ -0,0 +1,39 @@
+# Copyright (c) 2026 BAAI. All rights reserved.
+
+"""Tests for the DeepSeek-V4 attention compile boundary."""
+
+from types import SimpleNamespace
+
+import torch
+
+from vllm_fl.models import deepseek_v4
+
+
+def test_deepseek_v4_fl_attention_writes_preallocated_output(monkeypatch):
+    calls = []
+
+    class Layer:
+        def attention_impl(self, *args):
+            calls.append(args)
+            args[-1].fill_(7)
+
+    layer = Layer()
+    monkeypatch.setattr(
+        deepseek_v4,
+        "get_forward_context",
+        lambda: SimpleNamespace(no_compile_layers={"layer": layer}),
+    )
+
+    tensors = [torch.empty(1) for _ in range(7)]
+    out = torch.empty(2, 3, 4)
+    result = deepseek_v4._deepseek_v4_fl_attention(*tensors, out, "layer")
+
+    assert result is None
+    assert len(calls) == 1
+    assert calls[0] == (*tensors, out)
+    assert calls[0][-1] is out
+    assert torch.equal(out, torch.full_like(out, 7))
+    schema = torch._C._dispatch_find_schema_or_throw(
+        "vllm::deepseek_v4_fl_attention", ""
+    ).schema()
+    assert "Tensor(a7!) out" in str(schema)

tests/unit_tests/dispatch/test_deepseek_v4_ops.py

@@ -0,0 +1,218 @@
+# Copyright (c) 2026 BAAI. All rights reserved.
+
+"""Tests for DeepSeek-V4 operator dispatch."""
+
+from unittest.mock import Mock
+
+import torch
+
+from vllm_fl.dispatch.backends.reference.impl.deepseek_v4 import (
+    deepseek_v4_hc_head_torch,
+    deepseek_v4_int8_scaled_mm_torch,
+    deepseek_v4_inv_rope_quant_int8_torch,
+    deepseek_v4_mhc_post_torch,
+)
+from vllm_fl.dispatch.types import BackendImplKind
+from vllm_fl.ops import deepseek_v4_int8_woa
+
+DSV4_OPS = {
+    "deepseek_v4_inv_rope_quant_int8",
+    "deepseek_v4_inv_rope_quant_fp8",
+    "deepseek_v4_int8_scaled_mm",
+    "deepseek_v4_mhc_pre",
+    "deepseek_v4_mhc_fused_post_pre",
+    "deepseek_v4_mhc_post",
+    "deepseek_v4_hc_head",
+    "deepseek_v4_fused_q_kv_rmsnorm",
+    "deepseek_v4_qnorm_rope_kv_quant_insert",
+    "deepseek_v4_qnorm_rope_kv_bf16_insert",
+    "deepseek_v4_qnorm_rope_kv_fp8_insert",
+    "deepseek_v4_compute_global_topk_indices_and_lens",
+    "deepseek_v4_flash_mla_with_kvcache",
+    "deepseek_v4_dequantize_and_gather_k_cache",
+    "deepseek_v4_combine_topk_swa_indices",
+    "deepseek_v4_flash_mla_sparse_fwd",
+    "deepseek_v4_fused_indexer_q_rope_quant",
+    "deepseek_v4_fused_indexer_q_rope_quant_int8",
+    "deepseek_v4_compress_int8_indexer_k_cache",
+    "deepseek_v4_int8_mqa_logits",
+    "deepseek_v4_int8_paged_mqa_logits",
+}
+
+SPARSE_INDEXER_OPS = {
+    "indexer_k_quant_and_cache",
+    "cp_gather_indexer_k_quant_cache",
+    "top_k_per_row_prefill",
+    "top_k_per_row_decode",
+    "pack_seq_triton",
+    "unpack_seq_triton",
+}
+
+
+def test_reference_inv_rope_quant_int8():
+    o = torch.tensor(
+        [[[1, 2, 3, 4], [-1, -2, 5, 6]]],
+        dtype=torch.bfloat16,
+    )
+    positions = torch.tensor([0], dtype=torch.int32)
+    cos_sin_cache = torch.tensor([[0, 1]], dtype=torch.float32)
+
+    quantized, scales = deepseek_v4_inv_rope_quant_int8_torch(
+        o,
+        positions,
+        cos_sin_cache,
+        n_groups=1,
+        heads_per_group=2,
+        nope_dim=2,
+        rope_dim=2,
+    )
+
+    expected = torch.tensor(
+        [[[21, 42, 85, -64, -21, -42, 127, -106]]],
+        dtype=torch.int8,
+    )
+    assert torch.equal(quantized, expected)
+    torch.testing.assert_close(
+        scales,
+        torch.tensor([[[6 / 127]]], dtype=torch.float32),
+    )
+
+
+def test_frontend_dispatches_through_cached_op(monkeypatch):
+    expected = (Mock(), Mock())
+    dispatch = Mock(return_value=expected)
+    monkeypatch.setattr(
+        deepseek_v4_int8_woa,
+        "_dispatch_inv_rope_quant_int8",
+        dispatch,
+    )
+    args = (
+        Mock(),
+        Mock(),
+        Mock(),
+        2,
+        4,
+        64,
+        64,
+    )
+
+    actual = deepseek_v4_int8_woa.fused_inv_rope_quant_int8(*args)
+
+    assert actual is expected
+    dispatch.assert_called_once_with(*args)
+
+
+def test_all_backends_register_deepseek_v4_op(monkeypatch):
+    from vllm_fl.dispatch.backends.flaggems import register_ops as flaggems_ops
+    from vllm_fl.dispatch.backends.reference import register_ops as reference_ops
+    from vllm_fl.dispatch.backends.vendor.cuda import register_ops as cuda_ops
+
+    registered = []
+
+    class Registry:
+        def register_many(self, impls):
+            registered.extend(impls)
+
+    monkeypatch.setattr(
+        flaggems_ops,
+        "use_flaggems_op",
+        lambda op_name: op_name == deepseek_v4_int8_woa.DSV4_INV_ROPE_QUANT_INT8_OP,
+    )
+    registry = Registry()
+    flaggems_ops.register_builtins(registry)
+    cuda_ops.register_builtins(registry)
+    reference_ops.register_builtins(registry)
+
+    implementations = [
+        impl
+        for impl in registered
+        if impl.op_name == deepseek_v4_int8_woa.DSV4_INV_ROPE_QUANT_INT8_OP
+    ]
+    assert {impl.impl_id for impl in implementations} == {
+        "default.flagos",
+        "vendor.cuda",
+        "reference.torch",
+    }
+    assert {impl.kind for impl in implementations} == {
+        BackendImplKind.DEFAULT,
+        BackendImplKind.VENDOR,
+        BackendImplKind.REFERENCE,
+    }
+
+
+def test_reference_scaled_mm_and_mhc_ops():
+    x_q = torch.tensor([[1, -2]], dtype=torch.int8)
+    weight = torch.tensor([[3, 4], [5, 6]], dtype=torch.int8)
+    actual = deepseek_v4_int8_scaled_mm_torch(
+        x_q,
+        weight,
+        torch.tensor([[0.5]]),
+        torch.tensor([0.25, 0.5]),
+        torch.float32,
+    )
+    torch.testing.assert_close(actual, torch.tensor([[-0.875, -2.0]]))
+
+    residual = torch.tensor([[[1, 2], [3, 4]]], dtype=torch.bfloat16)
+    layer = torch.tensor([[2, -1]], dtype=torch.bfloat16)
+    post = torch.tensor([[[0.5], [1.0]]], dtype=torch.float32)
+    comb = torch.eye(2, dtype=torch.float32).unsqueeze(0)
+    torch.testing.assert_close(
+        deepseek_v4_mhc_post_torch(layer, residual, post, comb),
+        torch.tensor([[[2, 1.5], [5, 3]]], dtype=torch.bfloat16),
+    )
+
+    fn = torch.zeros((2, 4), dtype=torch.float32)
+    head = deepseek_v4_hc_head_torch(
+        residual,
+        fn,
+        torch.ones(1),
+        torch.zeros(2),
+        1e-6,
+        0.0,
+    )
+    torch.testing.assert_close(head, residual.float().mean(dim=1).to(torch.bfloat16))
+
+
+def test_all_backends_register_all_deepseek_v4_ops(monkeypatch):
+    from vllm_fl.dispatch.backends.flaggems import register_ops as flaggems_ops
+    from vllm_fl.dispatch.backends.reference import register_ops as reference_ops
+    from vllm_fl.dispatch.backends.vendor.cuda import register_ops as cuda_ops
+
+    registered = []
+
+    class Registry:
+        def register_many(self, impls):
+            registered.extend(impls)
+
+    monkeypatch.setattr(
+        flaggems_ops,
+        "use_flaggems_op",
+        lambda op_name: op_name in DSV4_OPS | SPARSE_INDEXER_OPS,
+    )
+    registry = Registry()
+    flaggems_ops.register_builtins(registry)
+    cuda_ops.register_builtins(registry)
+    reference_ops.register_builtins(registry)
+
+    for op_name in DSV4_OPS:
+        implementations = [impl for impl in registered if impl.op_name == op_name]
+        assert {impl.impl_id for impl in implementations} == {
+            "default.flagos",
+            "vendor.cuda",
+            "reference.torch",
+        }
+
+    for op_name in SPARSE_INDEXER_OPS:
+        implementations = [impl for impl in registered if impl.op_name == op_name]
+        assert {impl.impl_id for impl in implementations} == {
+            "default.flagos",
+            "vendor.cuda",
+            "reference.torch",
+        }
+
+
+def test_sparse_indexer_overrides_upstream_cuda_entrypoint():
+    from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
+    from vllm_fl.ops.sparse_attn_indexer import SparseAttnIndexerFL
+
+    assert SparseAttnIndexerFL.forward_cuda is not SparseAttnIndexer.forward_cuda